[pinot] branch cancel-request updated (d66cc7e281 -> 1f412059e5)
This is an automated email from the ASF dual-hosted git repository. kishoreg pushed a change to branch cancel-request in repository https://gitbox.apache.org/repos/asf/pinot.git discard d66cc7e281 Initial commit to support cancelling a long running query add 1f412059e5 Initial commit to support cancelling a long running query This update added new revisions after undoing existing revisions. That is to say, some revisions that were in the old version of the branch are not in the new version. This situation occurs when a user --force pushes a change and generates a repository containing something like this: * -- * -- B -- O -- O -- O (d66cc7e281) \ N -- N -- N refs/heads/cancel-request (1f412059e5) You should already have received notification emails for all of the O revisions, and so the following emails describe only the N revisions from the common base, B. Any revisions marked "omit" are not gone; other references still refer to them. Any revisions marked "discard" are gone forever. No new revisions were added by this update. Summary of changes: .../src/main/java/org/apache/pinot/core/operator/BaseOperator.java | 7 --- 1 file changed, 7 deletions(-) - To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
[pinot] branch cancel-request created (now d66cc7e281)
This is an automated email from the ASF dual-hosted git repository. kishoreg pushed a change to branch cancel-request in repository https://gitbox.apache.org/repos/asf/pinot.git at d66cc7e281 Initial commit to support cancelling a long running query This branch includes the following new commits: new d66cc7e281 Initial commit to support cancelling a long running query The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. - To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
[pinot] 01/01: Initial commit to support cancelling a long running query
This is an automated email from the ASF dual-hosted git repository.
kishoreg pushed a commit to branch cancel-request
in repository https://gitbox.apache.org/repos/asf/pinot.git
commit d66cc7e2815939d6ba462e5182a71e6eff8e4c5a
Author: kishoreg
AuthorDate: Sun Jul 31 23:50:56 2022 -0700
Initial commit to support cancelling a long running query
---
.../broker/api/resources/PinotClientRequest.java | 16 +
.../requesthandler/BaseBrokerRequestHandler.java | 23 ++-
.../requesthandler/BrokerRequestHandler.java | 2 +
.../BrokerRequestHandlerDelegate.java | 6 +-
.../apache/pinot/core/operator/BaseOperator.java | 7 ++
.../core/operator/combine/BaseCombineOperator.java | 8 +++
.../apache/pinot/core/util/trace/TraceContext.java | 23 +++
.../pinot/server/api/resources/QueryResource.java | 80 ++
8 files changed, 163 insertions(+), 2 deletions(-)
diff --git
a/pinot-broker/src/main/java/org/apache/pinot/broker/api/resources/PinotClientRequest.java
b/pinot-broker/src/main/java/org/apache/pinot/broker/api/resources/PinotClientRequest.java
index 9bc1b466c0..376a847256 100644
---
a/pinot-broker/src/main/java/org/apache/pinot/broker/api/resources/PinotClientRequest.java
+++
b/pinot-broker/src/main/java/org/apache/pinot/broker/api/resources/PinotClientRequest.java
@@ -115,6 +115,22 @@ public class PinotClientRequest {
}
}
+ @GET
+ @ManagedAsync
+ @Produces(MediaType.APPLICATION_JSON)
+ @Path("query/cancel")
+ @ApiOperation(value = "Querying pinot")
+ @ApiResponses(value = {
+ @ApiResponse(code = 200, message = "Query response"),
+ @ApiResponse(code = 500, message = "Internal Server Error")
+ })
+ public void cancelSqlQuery(@ApiParam(value = "RequestId", required = true)
@QueryParam("RequestId") long requestId,
+ @Suspended AsyncResponse asyncResponse, @Context
org.glassfish.grizzly.http.server.Request requestContext) {
+//todo: should we get the query as well.. probably not needed if the
broker maintains requestId <> RequestContext for the running queries
+String response = _requestHandler.cancelRequest(requestId);
+asyncResponse.resume(response);
+
+ }
@POST
@ManagedAsync
@Produces(MediaType.APPLICATION_JSON)
diff --git
a/pinot-broker/src/main/java/org/apache/pinot/broker/requesthandler/BaseBrokerRequestHandler.java
b/pinot-broker/src/main/java/org/apache/pinot/broker/requesthandler/BaseBrokerRequestHandler.java
index 3f2bd37322..9c91eb8572 100644
---
a/pinot-broker/src/main/java/org/apache/pinot/broker/requesthandler/BaseBrokerRequestHandler.java
+++
b/pinot-broker/src/main/java/org/apache/pinot/broker/requesthandler/BaseBrokerRequestHandler.java
@@ -31,6 +31,7 @@ import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicInteger;
@@ -125,6 +126,7 @@ public abstract class BaseBrokerRequestHandler implements
BrokerRequestHandler {
private final boolean _enableQueryLimitOverride;
private final boolean _enableDistinctCountBitmapOverride;
+
public BaseBrokerRequestHandler(PinotConfiguration config,
BrokerRoutingManager routingManager,
AccessControlFactory accessControlFactory, QueryQuotaManager
queryQuotaManager, TableCache tableCache,
BrokerMetrics brokerMetrics) {
@@ -189,7 +191,23 @@ public abstract class BaseBrokerRequestHandler implements
BrokerRequestHandler {
if (sql == null) {
throw new BadQueryRequestException("Failed to find 'sql' in the request:
" + request);
}
-return handleRequest(requestId, sql.asText(), request, requesterIdentity,
requestContext);
+BrokerResponseNative brokerResponseNative =
+handleRequest(requestId, sql.asText(), request, requesterIdentity,
requestContext);
+return brokerResponseNative;
+ }
+
+ @Override
+ public String cancelRequest(long requestId) {
+try {
+
+ //we will send the cancel request to all servers.. we can probably
optimize by asking the user to provide the query again or the list of tables
+ //Assuming this wont be frequently called.. invoking this on all servers
it not a big overhead
+
+ return "";
+
+} catch (Exception e) {
+ return "Exception while trying to cancel request: " + requestId + ". " +
e.getMessage();
+}
}
private BrokerResponseNative handleRequest(long requestId, String query,
JsonNode request,
@@ -1472,6 +1490,9 @@ public abstract class BaseBrokerRequestHandler implements
BrokerRequestHandler {
if (enableTrace) {
queryOptions.put(Broker.Request.TRACE, "true");
}
+
+queryOptions.put(CommonConstants.Query.Request.MetadataKeys.REQUEST_ID,
String.value
[pinot] branch message-header updated (b52d0659b6 -> bfd18f9b14)
This is an automated email from the ASF dual-hosted git repository. kishoreg pushed a change to branch message-header in repository https://gitbox.apache.org/repos/asf/pinot.git from b52d0659b6 Adding few udf to convert byte[] to string and also prefixing the header fields with header$ prefix to seperate them from the keys in message payload add bfd18f9b14 adding null check for row metadata No new revisions were added by this update. Summary of changes: .../pinot/core/data/manager/realtime/LLRealtimeSegmentDataManager.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) - To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
[pinot] branch message-header updated (f89cc3b53c -> b52d0659b6)
This is an automated email from the ASF dual-hosted git repository. kishoreg pushed a change to branch message-header in repository https://gitbox.apache.org/repos/asf/pinot.git from f89cc3b53c Adding support to extract values from message header.. initial support for kafka headers add b52d0659b6 Adding few udf to convert byte[] to string and also prefixing the header fields with header$ prefix to seperate them from the keys in message payload No new revisions were added by this update. Summary of changes: .../common/function/scalar/StringFunctions.java| 41 +- .../realtime/LLRealtimeSegmentDataManager.java | 2 +- .../tests/BaseClusterIntegrationTest.java | 5 ++- .../tests/ClusterIntegrationTestUtils.java | 7 +++- .../tests/RealtimeClusterIntegrationTest.java | 12 +++ .../stream/kafka20/RowMetadataExtractor.java | 2 +- .../stream/kafka20/server/KafkaDataProducer.java | 20 +++ .../pinot/spi/stream/StreamDataProducer.java | 9 +++-- 8 files changed, 91 insertions(+), 7 deletions(-) - To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
[pinot] branch message-header created (now f89cc3b53c)
This is an automated email from the ASF dual-hosted git repository. kishoreg pushed a change to branch message-header in repository https://gitbox.apache.org/repos/asf/pinot.git at f89cc3b53c Adding support to extract values from message header.. initial support for kafka headers This branch includes the following new commits: new f89cc3b53c Adding support to extract values from message header.. initial support for kafka headers The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. - To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
[pinot] 01/01: Adding support to extract values from message header.. initial support for kafka headers
This is an automated email from the ASF dual-hosted git repository.
kishoreg pushed a commit to branch message-header
in repository https://gitbox.apache.org/repos/asf/pinot.git
commit f89cc3b53c52b2481575f92c1245aab844e6ea22
Author: kishoreg
AuthorDate: Sat Jul 23 11:47:30 2022 -0700
Adding support to extract values from message header.. initial support for
kafka headers
---
.../realtime/LLRealtimeSegmentDataManager.java | 8 +++-
.../kafka20/KafkaPartitionLevelConsumer.java | 13 +++-
.../stream/kafka20/RowMetadataExtractor.java | 23 +-
.../org/apache/pinot/spi/stream/RowMetadata.java | 6 ++
.../pinot/spi/stream/StreamMessageMetadata.java| 14 +
5 files changed, 61 insertions(+), 3 deletions(-)
diff --git
a/pinot-core/src/main/java/org/apache/pinot/core/data/manager/realtime/LLRealtimeSegmentDataManager.java
b/pinot-core/src/main/java/org/apache/pinot/core/data/manager/realtime/LLRealtimeSegmentDataManager.java
index fdfe31377e..1de9c818ba 100644
---
a/pinot-core/src/main/java/org/apache/pinot/core/data/manager/realtime/LLRealtimeSegmentDataManager.java
+++
b/pinot-core/src/main/java/org/apache/pinot/core/data/manager/realtime/LLRealtimeSegmentDataManager.java
@@ -527,8 +527,14 @@ public class LLRealtimeSegmentDataManager extends
RealtimeSegmentDataManager {
RowMetadata msgMetadata = messagesAndOffsets.getMetadataAtIndex(index);
GenericRow decodedRow = _messageDecoder
- .decode(messagesAndOffsets.getMessageAtIndex(index),
messagesAndOffsets.getMessageOffsetAtIndex(index),
+ .decode(messagesAndOffsets.getMessageAtIndex(index),
+ messagesAndOffsets.getMessageOffsetAtIndex(index),
messagesAndOffsets.getMessageLengthAtIndex(index), reuse);
+ if (msgMetadata.getHeaders() != null) {
+for (Map.Entry entrySet :
msgMetadata.getHeaders().getFieldToValueMap().entrySet()) {
+ decodedRow.putValue(entrySet.getKey(), entrySet.getValue());
+}
+ }
if (decodedRow != null) {
try {
_transformPipeline.processRow(decodedRow, reusedResult);
diff --git
a/pinot-plugins/pinot-stream-ingestion/pinot-kafka-2.0/src/main/java/org/apache/pinot/plugin/stream/kafka20/KafkaPartitionLevelConsumer.java
b/pinot-plugins/pinot-stream-ingestion/pinot-kafka-2.0/src/main/java/org/apache/pinot/plugin/stream/kafka20/KafkaPartitionLevelConsumer.java
index bf212cd855..31292cc09c 100644
---
a/pinot-plugins/pinot-stream-ingestion/pinot-kafka-2.0/src/main/java/org/apache/pinot/plugin/stream/kafka20/KafkaPartitionLevelConsumer.java
+++
b/pinot-plugins/pinot-stream-ingestion/pinot-kafka-2.0/src/main/java/org/apache/pinot/plugin/stream/kafka20/KafkaPartitionLevelConsumer.java
@@ -23,11 +23,16 @@ import java.util.ArrayList;
import java.util.List;
import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.apache.kafka.clients.consumer.ConsumerRecords;
+import org.apache.kafka.common.header.Header;
+import org.apache.kafka.common.header.Headers;
import org.apache.kafka.common.utils.Bytes;
+import org.apache.pinot.spi.data.readers.GenericRow;
import org.apache.pinot.spi.stream.LongMsgOffset;
import org.apache.pinot.spi.stream.MessageBatch;
import org.apache.pinot.spi.stream.PartitionLevelConsumer;
+import org.apache.pinot.spi.stream.RowMetadata;
import org.apache.pinot.spi.stream.StreamConfig;
+import org.apache.pinot.spi.stream.StreamMessageMetadata;
import org.apache.pinot.spi.stream.StreamPartitionMsgOffset;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -38,6 +43,8 @@ public class KafkaPartitionLevelConsumer extends
KafkaPartitionLevelConnectionHa
private static final Logger LOGGER =
LoggerFactory.getLogger(KafkaPartitionLevelConsumer.class);
+ GenericRow _headers = new GenericRow();
+
public KafkaPartitionLevelConsumer(String clientId, StreamConfig
streamConfig, int partition) {
super(clientId, streamConfig, partition);
}
@@ -65,8 +72,12 @@ public class KafkaPartitionLevelConsumer extends
KafkaPartitionLevelConnectionHa
long offset = messageAndOffset.offset();
if (offset >= startOffset & (endOffset > offset | endOffset == -1)) {
if (message != null) {
+ RowMetadata rowMetadata = null;
+ if (_config.isPopulateMetadata()) {
+rowMetadata = _rowMetadataExtractor.extract(messageAndOffset);
+ }
filtered.add(
- new MessageAndOffsetAndMetadata(message.get(), offset,
_rowMetadataExtractor.extract(messageAndOffset)));
+ new MessageAndOffsetAndMetadata(message.get(), offset,
rowMetadata));
} else if (LOGGER.isDebugEnabled()) {
LOGGER.debug("tombstone message at offset {}", offset);
}
diff --git
a/pinot-plugins/pinot-stream-ingestion/pinot-kafka-2.0/src/main/java/org/apache/pinot/plugin/stream/kafka20/RowMetadataExtractor.java
b/pinot-
[pinot] branch master updated: fix: Query console goes blank on syntax error (#8006)
This is an automated email from the ASF dual-hosted git repository.
kishoreg pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 44c4b65 fix: Query console goes blank on syntax error (#8006)
44c4b65 is described below
commit 44c4b65b990e4e05dbca973f3d7073c89dc3dc65
Author: Sanket Shah
AuthorDate: Thu Jan 13 02:15:55 2022 +0530
fix: Query console goes blank on syntax error (#8006)
---
pinot-controller/src/main/resources/app/components/Table.tsx | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/pinot-controller/src/main/resources/app/components/Table.tsx
b/pinot-controller/src/main/resources/app/components/Table.tsx
index a054a8c..22a8462 100644
--- a/pinot-controller/src/main/resources/app/components/Table.tsx
+++ b/pinot-controller/src/main/resources/app/components/Table.tsx
@@ -521,7 +521,7 @@ export default function CustomizedTables({
className={isCellClickable ?
classes.isCellClickable : (isSticky ? classes.isSticky : '')}
onClick={() => {cellClickCallback &&
cellClickCallback(cell);}}
>
-{makeCell(cell, index)}
+{makeCell(cell || '--', index)}
);
})}
-
To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org
For additional commands, e-mail: commits-h...@pinot.apache.org
[pinot] branch master updated: log query exception by using correct slf4j overload (#7948)
This is an automated email from the ASF dual-hosted git repository. kishoreg pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/pinot.git The following commit(s) were added to refs/heads/master by this push: new f2eb2fd log query exception by using correct slf4j overload (#7948) f2eb2fd is described below commit f2eb2fd824b6576f21a67b13ff6c6e654b7968d0 Author: Richard Startin AuthorDate: Wed Dec 22 17:53:18 2021 + log query exception by using correct slf4j overload (#7948) --- .../org/apache/pinot/core/operator/combine/BaseCombineOperator.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pinot-core/src/main/java/org/apache/pinot/core/operator/combine/BaseCombineOperator.java b/pinot-core/src/main/java/org/apache/pinot/core/operator/combine/BaseCombineOperator.java index 17ffb3b..2a3ac72 100644 --- a/pinot-core/src/main/java/org/apache/pinot/core/operator/combine/BaseCombineOperator.java +++ b/pinot-core/src/main/java/org/apache/pinot/core/operator/combine/BaseCombineOperator.java @@ -102,7 +102,7 @@ public abstract class BaseCombineOperator extends BaseOperator
[pinot] branch pub-sub created (now a485eaf)
This is an automated email from the ASF dual-hosted git repository. kishoreg pushed a change to branch pub-sub in repository https://gitbox.apache.org/repos/asf/pinot.git. at a485eaf Interface changes needed to support pub-sub This branch includes the following new commits: new a485eaf Interface changes needed to support pub-sub The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. - To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
[pinot] 01/01: Interface changes needed to support pub-sub
This is an automated email from the ASF dual-hosted git repository.
kishoreg pushed a commit to branch pub-sub
in repository https://gitbox.apache.org/repos/asf/pinot.git
commit a485eaf0a462a6003d7313d4417a1ad17311b63f
Author: kishoreg
AuthorDate: Thu Nov 25 09:14:05 2021 -1000
Interface changes needed to support pub-sub
---
.../realtime/LLRealtimeSegmentDataManager.java | 341 ++---
.../pinot/spi/stream/PartitionGroupConsumer.java | 19 +-
2 files changed, 247 insertions(+), 113 deletions(-)
diff --git
a/pinot-core/src/main/java/org/apache/pinot/core/data/manager/realtime/LLRealtimeSegmentDataManager.java
b/pinot-core/src/main/java/org/apache/pinot/core/data/manager/realtime/LLRealtimeSegmentDataManager.java
index a36935d..a7dc545 100644
---
a/pinot-core/src/main/java/org/apache/pinot/core/data/manager/realtime/LLRealtimeSegmentDataManager.java
+++
b/pinot-core/src/main/java/org/apache/pinot/core/data/manager/realtime/LLRealtimeSegmentDataManager.java
@@ -7,7 +7,7 @@
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
@@ -94,11 +94,12 @@ import org.joda.time.DateTimeZone;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-
/**
- * Segment data manager for low level consumer realtime segments, which
manages consumption and segment completion.
+ * Segment data manager for low level consumer realtime segments, which
manages consumption and
+ * segment completion.
*/
public class LLRealtimeSegmentDataManager extends RealtimeSegmentDataManager {
+
protected enum State {
// The state machine starts off with this state. While in this state we
consume stream events
// and index them in memory. We continue to be in this state until the end
criteria is satisfied
@@ -140,11 +141,13 @@ public class LLRealtimeSegmentDataManager extends
RealtimeSegmentDataManager {
ERROR;
public boolean shouldConsume() {
- return this.equals(INITIAL_CONSUMING) || this.equals(CATCHING_UP) ||
this.equals(CONSUMING_TO_ONLINE);
+ return this.equals(INITIAL_CONSUMING) || this.equals(CATCHING_UP) || this
+ .equals(CONSUMING_TO_ONLINE);
}
public boolean isFinal() {
- return this.equals(ERROR) || this.equals(COMMITTED) ||
this.equals(RETAINED) || this.equals(DISCARDED);
+ return this.equals(ERROR) || this.equals(COMMITTED) ||
this.equals(RETAINED) || this
+ .equals(DISCARDED);
}
}
@@ -152,6 +155,7 @@ public class LLRealtimeSegmentDataManager extends
RealtimeSegmentDataManager {
@VisibleForTesting
public class SegmentBuildDescriptor {
+
final File _segmentTarFile;
final Map _metadataFileMap;
final StreamPartitionMsgOffset _offset;
@@ -159,8 +163,10 @@ public class LLRealtimeSegmentDataManager extends
RealtimeSegmentDataManager {
final long _buildTimeMillis;
final long _segmentSizeBytes;
-public SegmentBuildDescriptor(@Nullable File segmentTarFile, @Nullable
Map metadataFileMap,
-StreamPartitionMsgOffset offset, long buildTimeMillis, long
waitTimeMillis, long segmentSizeBytes) {
+public SegmentBuildDescriptor(@Nullable File segmentTarFile,
+@Nullable Map metadataFileMap,
+StreamPartitionMsgOffset offset, long buildTimeMillis, long
waitTimeMillis,
+long segmentSizeBytes) {
_segmentTarFile = segmentTarFile;
_metadataFileMap = metadataFileMap;
_offset = _streamPartitionMsgOffsetFactory.create(offset);
@@ -301,23 +307,27 @@ public class LLRealtimeSegmentDataManager extends
RealtimeSegmentDataManager {
// the max time we are allowed to consume.
if (now >= _consumeEndTime) {
if (_realtimeSegment.getNumDocsIndexed() == 0) {
-_segmentLogger.info("No events came in, extending time by {}
hours", TIME_EXTENSION_ON_EMPTY_SEGMENT_HOURS);
+_segmentLogger.info("No events came in, extending time by {}
hours",
+TIME_EXTENSION_ON_EMPTY_SEGMENT_HOURS);
_consumeEndTime +=
TimeUnit.HOURS.toMillis(TIME_EXTENSION_ON_EMPTY_SEGMENT_HOURS);
return false;
}
_segmentLogger
- .info("Stopping consumption due to time limit start={} now={}
numRowsConsumed={} numRowsIndexed={}",
+ .info(
+ "Stopping consumption due to time limit start={} now={}
numRowsConsumed={} numRowsIndexed={}",
_startTimeMs, now, _numRowsConsumed, _numRowsIndexed);
_stopReason = SegmentCompletionProtocol.REASON_TIME_LIMIT;
return true;
} else if (_numRowsIndexed >= _segmentMaxRowCount) {
-
[pinot] branch master updated: remove unused Number2ObjectPair class and auxiliaries (#7736)
This is an automated email from the ASF dual-hosted git repository.
kishoreg pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new b66643a remove unused Number2ObjectPair class and auxiliaries (#7736)
b66643a is described below
commit b66643aa671c8b21b6e6320b25fed1a03e15d1fc
Author: Richard Startin
AuthorDate: Wed Nov 10 18:10:14 2021 +
remove unused Number2ObjectPair class and auxiliaries (#7736)
---
.../java/org/apache/pinot/spi/utils/Pairs.java | 44 +-
1 file changed, 2 insertions(+), 42 deletions(-)
diff --git a/pinot-spi/src/main/java/org/apache/pinot/spi/utils/Pairs.java
b/pinot-spi/src/main/java/org/apache/pinot/spi/utils/Pairs.java
index d3e8a11..be18d35 100644
--- a/pinot-spi/src/main/java/org/apache/pinot/spi/utils/Pairs.java
+++ b/pinot-spi/src/main/java/org/apache/pinot/spi/utils/Pairs.java
@@ -90,46 +90,6 @@ public class Pairs {
}
}
- public static Comparator
getAscendingnumber2ObjectPairComparator() {
-return new AscendingNumber2ObjectPairComparator();
- }
-
- public static Comparator
getDescendingnumber2ObjectPairComparator() {
-return new DescendingNumber2ObjectPairComparator();
- }
-
- public static class Number2ObjectPair {
-Number _a;
-T _b;
-
-public Number2ObjectPair(Number a, T b) {
- _a = a;
- _b = b;
-}
-
-public Number getA() {
- return _a;
-}
-
-public T getB() {
- return _b;
-}
- }
-
- public static class AscendingNumber2ObjectPairComparator implements
Comparator {
-@Override
-public int compare(Number2ObjectPair o1, Number2ObjectPair o2) {
- return new Double(o1._a.doubleValue()).compareTo(new
Double(o2._a.doubleValue()));
-}
- }
-
- public static class DescendingNumber2ObjectPairComparator implements
Comparator {
-@Override
-public int compare(Number2ObjectPair o1, Number2ObjectPair o2) {
- return new Double(o2._a.doubleValue()).compareTo(new
Double(o1._a.doubleValue()));
-}
- }
-
/**
* Utility class to store a primitive 'int' and 'double' pair.
*/
@@ -273,9 +233,9 @@ public class Pairs {
Comparable c2 = (Comparable) pair2.getObjectValue();
int cmpValue = c1.compareTo(c2);
- if (cmpValue == -1) {
+ if (cmpValue < 0) {
return (_descending) ? 1 : -1;
- } else if (cmpValue == 1) {
+ } else if (cmpValue > 0) {
return (_descending) ? -1 : 1;
} else {
return 0;
-
To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org
For additional commands, e-mail: commits-h...@pinot.apache.org
[pinot] branch master updated: Add MV raw forward index and MV `BYTES` data type (#7595)
This is an automated email from the ASF dual-hosted git repository.
kishoreg pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new aed1307 Add MV raw forward index and MV `BYTES` data type (#7595)
aed1307 is described below
commit aed13072dac0d8dae29056fac77f0f457be7adba
Author: Richard Startin
AuthorDate: Fri Oct 22 15:50:36 2021 +0100
Add MV raw forward index and MV `BYTES` data type (#7595)
* Initial code for MultiValue forward Index
* Wiring in the segment creation driver Impl
* cleanup
* finish off adding BYTES_ARRAY type
* use less memory and fewer passes during encoding
* reduce memory requirement for forwardindexwriter
* track size in bytes of largest row so chunks can be sized to accommodate
it
* remove TODOs
* force derivation of number of docs for raw MV columns
* specify character encoding
* leave changes to integration tests to MV TEXT index implementation
* fix javadoc
* don't use StringUtils
* fix formatting after rebase
* fix javadoc formatting again
* use zstd's compress bound
Co-authored-by: kishoreg
---
.../org/apache/pinot/common/utils/DataSchema.java | 9 +-
.../apache/pinot/common/utils/PinotDataType.java | 37 +++-
.../apache/pinot/common/utils/DataSchemaTest.java | 19 +-
.../pinot/common/utils/PinotDataTypeTest.java | 6 +-
.../pinot/core/minion/RawIndexConverter.java | 2 +-
.../local/io/compression/LZ4Compressor.java| 5 +
.../io/compression/PassThroughCompressor.java | 5 +
.../local/io/compression/SnappyCompressor.java | 5 +
.../local/io/compression/ZstandardCompressor.java | 5 +
.../writer/impl/BaseChunkSVForwardIndexWriter.java | 60 ---
.../impl/FixedByteChunkSVForwardIndexWriter.java | 4 +-
.../impl/VarByteChunkSVForwardIndexWriter.java | 77 ++--
.../creator/impl/SegmentColumnarIndexCreator.java | 192 +---
.../fwd/MultiValueFixedByteRawIndexCreator.java| 181 +++
.../impl/fwd/MultiValueVarByteRawIndexCreator.java | 122 +
.../stats/AbstractColumnStatisticsCollector.java | 5 +
.../stats/BytesColumnPredIndexStatsCollector.java | 44 +++--
.../stats/StringColumnPreIndexStatsCollector.java | 10 ++
.../forward/VarByteChunkMVForwardIndexReader.java | 193 +
.../local/segment/store/FilePerIndexDirectory.java | 6 +-
.../MultiValueVarByteRawIndexCreatorTest.java | 141 +++
.../segment/index/creator/RawIndexCreatorTest.java | 135 +++---
.../org/apache/pinot/segment/spi/V1Constants.java | 1 +
.../segment/spi/compression/ChunkCompressor.java | 2 +
.../spi/creator/ColumnIndexCreationInfo.java | 4 +
.../segment/spi/creator/ColumnStatistics.java | 7 +
.../spi/index/creator/ForwardIndexCreator.java | 9 +
.../spi/index/reader/ForwardIndexReader.java | 19 ++
.../converter/DictionaryToRawIndexConverter.java | 2 +-
29 files changed, 1187 insertions(+), 120 deletions(-)
diff --git
a/pinot-common/src/main/java/org/apache/pinot/common/utils/DataSchema.java
b/pinot-common/src/main/java/org/apache/pinot/common/utils/DataSchema.java
index 6b61cfc..37fb392 100644
--- a/pinot-common/src/main/java/org/apache/pinot/common/utils/DataSchema.java
+++ b/pinot-common/src/main/java/org/apache/pinot/common/utils/DataSchema.java
@@ -255,12 +255,13 @@ public class DataSchema {
DOUBLE_ARRAY,
BOOLEAN_ARRAY /* Stored as INT_ARRAY */,
TIMESTAMP_ARRAY /* Stored as LONG_ARRAY */,
+BYTES_ARRAY,
STRING_ARRAY;
private static final EnumSet NUMERIC_TYPES =
EnumSet.of(INT, LONG, FLOAT, DOUBLE);
private static final EnumSet INTEGRAL_TYPES =
EnumSet.of(INT, LONG);
private static final EnumSet ARRAY_TYPES =
EnumSet.of(INT_ARRAY, LONG_ARRAY, FLOAT_ARRAY,
-DOUBLE_ARRAY, STRING_ARRAY, BOOLEAN_ARRAY, TIMESTAMP_ARRAY);
+DOUBLE_ARRAY, STRING_ARRAY, BOOLEAN_ARRAY, TIMESTAMP_ARRAY,
BYTES_ARRAY);
private static final EnumSet NUMERIC_ARRAY_TYPES =
EnumSet.of(INT_ARRAY, LONG_ARRAY, FLOAT_ARRAY,
DOUBLE_ARRAY);
private static final EnumSet INTEGRAL_ARRAY_TYPES =
EnumSet.of(INT_ARRAY, LONG_ARRAY);
@@ -368,6 +369,8 @@ public class DataSchema {
return toBooleanArray(value);
case TIMESTAMP_ARRAY:
return toTimestampArray(value);
+case BYTES_ARRAY:
+ return (byte[][]) value;
default:
throw new IllegalStateException(String.format("Cannot convert: '%s'
to type: %s", value, this));
}
@@ -424,6 +427,8 @@ public class DataSchema {
return toBooleanArray(value);
case TIMESTAMP_ARRAY:
return formatTimestampArray(val
[pinot] branch mv-fwd-index updated: Wiring in the segment creation driver Impl
This is an automated email from the ASF dual-hosted git repository.
kishoreg pushed a commit to branch mv-fwd-index
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/mv-fwd-index by this push:
new e8bca30 Wiring in the segment creation driver Impl
e8bca30 is described below
commit e8bca30473c28e9e9efe371bd06442955da7d433
Author: kishoreg
AuthorDate: Sun Oct 17 20:46:09 2021 -0700
Wiring in the segment creation driver Impl
---
.../apache/pinot/common/utils/PinotDataType.java | 47 ++-
.../pinot/core/minion/RawIndexConverter.java | 2 +-
.../tests/BaseClusterIntegrationTest.java | 3 +-
.../impl/VarByteChunkSVForwardIndexWriter.java | 43 ++-
.../creator/impl/SegmentColumnarIndexCreator.java | 422 -
.../fwd/MultiValueFixedByteRawIndexCreator.java| 22 +-
.../impl/fwd/MultiValueVarByteRawIndexCreator.java | 23 +-
.../stats/BytesColumnPredIndexStatsCollector.java | 56 ++-
.../local/segment/store/FilePerIndexDirectory.java | 7 +-
.../MultiValueVarByteRawIndexCreatorTest.java | 45 ++-
.../segment/index/creator/RawIndexCreatorTest.java | 147 +--
.../converter/DictionaryToRawIndexConverter.java | 2 +-
12 files changed, 627 insertions(+), 192 deletions(-)
diff --git
a/pinot-common/src/main/java/org/apache/pinot/common/utils/PinotDataType.java
b/pinot-common/src/main/java/org/apache/pinot/common/utils/PinotDataType.java
index 49406ba..a94fee9 100644
---
a/pinot-common/src/main/java/org/apache/pinot/common/utils/PinotDataType.java
+++
b/pinot-common/src/main/java/org/apache/pinot/common/utils/PinotDataType.java
@@ -7,7 +7,7 @@
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
@@ -29,7 +29,6 @@ import org.apache.pinot.spi.utils.BytesUtils;
import org.apache.pinot.spi.utils.JsonUtils;
import org.apache.pinot.spi.utils.TimestampUtils;
-
/**
* The PinotDataType enum represents the data type of a value in
a row from recordReader and provides
* utility methods to convert value across types if applicable.
@@ -583,7 +582,8 @@ public enum PinotDataType {
try {
return Base64.getDecoder().decode(value.toString());
} catch (Exception e) {
-throw new RuntimeException("Unable to convert JSON base64 encoded
string value to BYTES. Input value: " + value,
+throw new RuntimeException(
+"Unable to convert JSON base64 encoded string value to BYTES.
Input value: " + value,
e);
}
}
@@ -769,6 +769,12 @@ public enum PinotDataType {
return sourceType.toStringArray(value);
}
},
+ BYTES_ARRAY {
+@Override
+public byte[][] convert(Object value, PinotDataType sourceType) {
+ return sourceType.toBytesArray(value);
+}
+ },
OBJECT_ARRAY;
@@ -817,7 +823,8 @@ public enum PinotDataType {
return JsonUtils.objectToString(value);
} catch (Exception e) {
throw new RuntimeException(
-"Unable to convert " + value.getClass().getCanonicalName() + " to
JSON. Input value: " + value, e);
+"Unable to convert " + value.getClass().getCanonicalName() + " to
JSON. Input value: "
++ value, e);
}
}
}
@@ -1020,6 +1027,24 @@ public enum PinotDataType {
}
}
+ public byte[][] toBytesArray(Object value) {
+if (value instanceof byte[][]) {
+ return (byte[][]) value;
+}
+if (isSingleValue()) {
+ return new byte[][]{toBytes(value)};
+} else {
+ Object[] valueArray = toObjectArray(value);
+ int length = valueArray.length;
+ byte[][] bytesArray = new byte[length][];
+ PinotDataType singleValueType = getSingleValueType();
+ for (int i = 0; i < length; i++) {
+bytesArray[i] = singleValueType.toBytes(valueArray[i]);
+ }
+ return bytesArray;
+}
+ }
+
private static Object[] toObjectArray(Object array) {
Class componentType = array.getClass().getComponentType();
if (componentType.isPrimitive()) {
@@ -1042,7 +1067,8 @@ public enum PinotDataType {
}
public Object convert(Object value, PinotDataType sourceType) {
-throw new UnsupportedOperationException("Cannot convert value from " +
sourceType + " to " + this);
+throw new UnsupportedOperationException(
+"Cannot convert value from " + sourceType + " to " + this);
}
/**
@@ -1082,6 +1108,8 @@ public enum PinotDataType {
return DOUBLE;
case STRING_ARRAY:
return STRING;
+ case BYTES_ARRAY:
+
[pinot] branch mv-fwd-index created (now 10b8d0a)
This is an automated email from the ASF dual-hosted git repository. kishoreg pushed a change to branch mv-fwd-index in repository https://gitbox.apache.org/repos/asf/pinot.git. at 10b8d0a Initial code for MultiValue forward Index This branch includes the following new commits: new 10b8d0a Initial code for MultiValue forward Index The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. - To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
[pinot] 01/01: Initial code for MultiValue forward Index
This is an automated email from the ASF dual-hosted git repository.
kishoreg pushed a commit to branch mv-fwd-index
in repository https://gitbox.apache.org/repos/asf/pinot.git
commit 10b8d0ac8c8a4e911c5355335ed1e182d0c38543
Author: kishoreg
AuthorDate: Sun Oct 17 00:19:04 2021 -0700
Initial code for MultiValue forward Index
---
.../fwd/MultiValueFixedByteRawIndexCreator.java| 179 +
.../impl/fwd/MultiValueVarByteRawIndexCreator.java | 214 +
.../stats/AbstractColumnStatisticsCollector.java | 5 +
.../forward/VarByteChunkMVForwardIndexReader.java | 197 +++
.../MultiValueVarByteRawIndexCreatorTest.java | 81
.../org/apache/pinot/segment/spi/V1Constants.java | 1 +
.../spi/index/creator/ForwardIndexCreator.java | 9 +
.../spi/index/reader/ForwardIndexReader.java | 19 ++
8 files changed, 705 insertions(+)
diff --git
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/fwd/MultiValueFixedByteRawIndexCreator.java
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/fwd/MultiValueFixedByteRawIndexCreator.java
new file mode 100644
index 000..d608a65
--- /dev/null
+++
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/fwd/MultiValueFixedByteRawIndexCreator.java
@@ -0,0 +1,179 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.segment.local.segment.creator.impl.fwd;
+
+import com.google.common.annotations.VisibleForTesting;
+import java.io.File;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import org.apache.commons.io.FileUtils;
+import
org.apache.pinot.segment.local.io.writer.impl.BaseChunkSVForwardIndexWriter;
+import
org.apache.pinot.segment.local.io.writer.impl.VarByteChunkSVForwardIndexWriter;
+import org.apache.pinot.segment.spi.V1Constants;
+import org.apache.pinot.segment.spi.compression.ChunkCompressionType;
+import org.apache.pinot.segment.spi.index.creator.ForwardIndexCreator;
+import org.apache.pinot.spi.data.FieldSpec.DataType;
+
+/**
+ * Forward index creator for raw (non-dictionary-encoded) single-value column
of variable length
+ * data type (STRING,
+ * BYTES).
+ */
+public class MultiValueFixedByteRawIndexCreator implements ForwardIndexCreator
{
+
+ private static final int DEFAULT_NUM_DOCS_PER_CHUNK = 1000;
+ private static final int TARGET_MAX_CHUNK_SIZE = 1024 * 1024;
+
+ private final VarByteChunkSVForwardIndexWriter _indexWriter;
+ private final DataType _valueType;
+
+ /**
+ * Create a var-byte raw index creator for the given column
+ *
+ * @param baseIndexDir Index directory
+ * @param compressionType Type of compression to use
+ * @param column Name of column to index
+ * @param totalDocs Total number of documents to index
+ * @param valueType Type of the values
+ * @param maxLength length of longest entry (in bytes)
+ */
+ public MultiValueFixedByteRawIndexCreator(File baseIndexDir,
ChunkCompressionType compressionType,
+ String column,
+ int totalDocs, DataType valueType, int maxLength)
+ throws IOException {
+this(baseIndexDir, compressionType, column, totalDocs, valueType,
maxLength, false,
+BaseChunkSVForwardIndexWriter.DEFAULT_VERSION);
+ }
+
+ /**
+ * Create a var-byte raw index creator for the given column
+ *
+ * @param baseIndexDir Index directory
+ * @param compressionType Type of compression to use
+ * @param column Name of column to index
+ * @param totalDocs Total number of documents to index
+ * @param valueType Type of the values
+ * @param maxLength length of longest entry (in bytes)
+ * @param deriveNumDocsPerChunk true if writer should auto-derive the number
of rows per chunk
+ * @param writerVersion writer format version
+ */
+ public MultiValueFixedByteRawIndexCreator(File baseIndexDir,
ChunkCompressionType compressionType,
+ String column,
+ int totalDocs, DataType valueType, int maxLength, boolean
deriveNumDocsPerChunk,
+ int writerVersion)
+ throws IOException {
+File file = new File(baseIndex
[pinot] branch master updated: Improved range queries (#7513)
This is an automated email from the ASF dual-hosted git repository.
kishoreg pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new af90f79 Improved range queries (#7513)
af90f79 is described below
commit af90f7972802f03b211c2c22becd8a838c46854b
Author: Richard Startin
AuthorDate: Tue Oct 5 17:08:40 2021 +0100
Improved range queries (#7513)
---
LICENSE-binary | 4 +--
.../org/apache/pinot/perf/BenchmarkRangeIndex.java | 15 +
.../index/column/PhysicalColumnIndexContainer.java | 2 +-
.../index/readers/BitSlicedRangeIndexReader.java | 39 --
.../index/creator/BitSlicedIndexCreatorTest.java | 16 ++---
pom.xml| 2 +-
6 files changed, 54 insertions(+), 24 deletions(-)
diff --git a/LICENSE-binary b/LICENSE-binary
index 08ee41e..c8c9569 100644
--- a/LICENSE-binary
+++ b/LICENSE-binary
@@ -379,8 +379,8 @@ org.eclipse.jetty:jetty-util:9.4.39.v20210325
org.javassist:javassist:3.19.0-GA
org.lz4:lz4-java:1.7.1
org.quartz-scheduler:quartz:2.3.2
-org.roaringbitmap:RoaringBitmap:0.9.21
-org.roaringbitmap:shims:0.9.21
+org.roaringbitmap:RoaringBitmap:0.9.22
+org.roaringbitmap:shims:0.9.22
org.webjars:swagger-ui:3.18.2
org.wildfly.openssl:wildfly-openssl:1.0.7.Final
org.xerial.larray:larray-buffer:0.4.1
diff --git
a/pinot-perf/src/main/java/org/apache/pinot/perf/BenchmarkRangeIndex.java
b/pinot-perf/src/main/java/org/apache/pinot/perf/BenchmarkRangeIndex.java
index ae02607..25079ca 100644
--- a/pinot-perf/src/main/java/org/apache/pinot/perf/BenchmarkRangeIndex.java
+++ b/pinot-perf/src/main/java/org/apache/pinot/perf/BenchmarkRangeIndex.java
@@ -196,7 +196,7 @@ public class BenchmarkRangeIndex {
@TearDown(Level.Trial)
public void tearDown()
throws IOException {
- FileUtils.forceDelete(_indexDir);
+ FileUtils.deleteQuietly(_indexDir);
}
protected Comparable max() {
@@ -378,19 +378,22 @@ public class BenchmarkRangeIndex {
public void setup()
throws IOException {
super.setup();
- _reader = new BitSlicedRangeIndexReader(_buffer);
+ _reader = new BitSlicedRangeIndexReader(_buffer, metadata());
}
-@Override
-protected RawValueBasedInvertedIndexCreator newCreator() {
- ColumnMetadata metadata = new ColumnMetadataImpl.Builder()
+private ColumnMetadata metadata() {
+ return new ColumnMetadataImpl.Builder()
.setFieldSpec(_fieldSpec)
.setTotalDocs(_numDocs)
.setHasDictionary(false)
.setMaxValue(max())
.setMinValue(min())
.build();
- return new BitSlicedRangeIndexCreator(_indexDir, metadata);
+}
+
+@Override
+protected RawValueBasedInvertedIndexCreator newCreator() {
+ return new BitSlicedRangeIndexCreator(_indexDir, metadata());
}
}
diff --git
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/column/PhysicalColumnIndexContainer.java
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/column/PhysicalColumnIndexContainer.java
index 49c7e2f..9ab1e7d 100644
---
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/column/PhysicalColumnIndexContainer.java
+++
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/column/PhysicalColumnIndexContainer.java
@@ -183,7 +183,7 @@ public final class PhysicalColumnIndexContainer implements
ColumnIndexContainer
if (version == RangeIndexCreator.VERSION) {
_rangeIndex = new RangeIndexReaderImpl(buffer);
} else if (version == BitSlicedRangeIndexCreator.VERSION) {
- _rangeIndex = new BitSlicedRangeIndexReader(buffer);
+ _rangeIndex = new BitSlicedRangeIndexReader(buffer, metadata);
} else {
LOGGER.warn("Unknown range index version: {}, skip loading range
index for column: {}", version,
metadata.getColumnName());
diff --git
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/BitSlicedRangeIndexReader.java
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/BitSlicedRangeIndexReader.java
index 032b4c3..41d34e1 100644
---
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/BitSlicedRangeIndexReader.java
+++
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/BitSlicedRangeIndexReader.java
@@ -23,11 +23,13 @@ import java.nio.ByteBuffer;
import javax.annotation.Nullable;
import
org.apache.pinot.segment.local.segment.creator.impl.inv.BitSlicedRangeIndexCreator;
import org.apache.pinot.segment.local.utils.FPOrdering;
+import org.apache.pinot.segment.spi.ColumnMetadat
[pinot] branch master updated (4972110 -> 5b15f08)
This is an automated email from the ASF dual-hosted git repository. kishoreg pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/pinot.git. from 4972110 Unify CombineOperator multi-threading logic (#7450) add 5b15f08 add SingleSpaceSeparator rule and fix recent violations (#7467) No new revisions were added by this update. Summary of changes: config/checkstyle.xml | 1 + 1 file changed, 1 insertion(+) - To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
[pinot-site] branch dev updated: change the Slack community inviter link for real
This is an automated email from the ASF dual-hosted git repository.
kishoreg pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/pinot-site.git
The following commit(s) were added to refs/heads/dev by this push:
new e577cd8 change the Slack community inviter link for real
new 5dd75cd Merge pull request #56 from
ananthdurai/replace_slack_community_inviter_for_all
e577cd8 is described below
commit e577cd81b46008978f95e1131e45fbd05e3645e4
Author: Ananth Packkildurai
AuthorDate: Thu Aug 12 09:04:27 2021 -0400
change the Slack community inviter link for real
---
README.md| 4 ++--
website/docusaurus.config.js | 2 +-
website/src/pages/index.js | 2 +-
3 files changed, 4 insertions(+), 4 deletions(-)
diff --git a/README.md b/README.md
index 5fb4075..191cdfe 100644
--- a/README.md
+++ b/README.md
@@ -1,7 +1,7 @@
Apache Pinot Site
=
-[](https://communityinviter.com/apps/apache-pinot/apache-pinot)
+[](https://join.slack.com/t/apache-pinot/shared_invite/zt-5z7pav2f-yYtjZdVA~EDmrGkho87Vzw)
[](https://twitter.com/intent/follow?screen_name=apachepinot)
[](LICENSE)
@@ -27,7 +27,7 @@ Additional Resources
Mailing list: d...@pinot.apache.org
-Slack: https://communityinviter.com/apps/apache-pinot/apache-pinot
+Slack:
https://join.slack.com/t/apache-pinot/shared_invite/zt-5z7pav2f-yYtjZdVA~EDmrGkho87Vzw
Project website: https://pinot.apache.org
diff --git a/website/docusaurus.config.js b/website/docusaurus.config.js
index 76c7b5a..941ed22 100755
--- a/website/docusaurus.config.js
+++ b/website/docusaurus.config.js
@@ -148,7 +148,7 @@ module.exports = {
items: [
{
label: 'Slack',
- to:
'https://communityinviter.com/apps/apache-pinot/apache-pinot',
+ to:
'https://join.slack.com/t/apache-pinot/shared_invite/zt-5z7pav2f-yYtjZdVA~EDmrGkho87Vzw',
},
{
label: 'Github',
diff --git a/website/src/pages/index.js b/website/src/pages/index.js
index 9709414..f1fb39e 100755
--- a/website/src/pages/index.js
+++ b/website/src/pages/index.js
@@ -476,7 +476,7 @@ function Home() {
Getting Started
https://communityinviter.com/apps/apache-pinot/apache-pinot";
+
to="https://join.slack.com/t/apache-pinot/shared_invite/zt-5z7pav2f-yYtjZdVA~EDmrGkho87Vzw";
className="button button--primary
button--highlight"
>
Join our Slack
-
To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org
For additional commands, e-mail: commits-h...@pinot.apache.org
[pinot-site] branch asf-site updated: updated Slack community link
This is an automated email from the ASF dual-hosted git repository. kishoreg pushed a commit to branch asf-site in repository https://gitbox.apache.org/repos/asf/pinot-site.git The following commit(s) were added to refs/heads/asf-site by this push: new 9ccceda updated Slack community link new b88329f Merge pull request #55 from ananthdurai/update_slack_community_link 9ccceda is described below commit 9ccceda575020be1c55cc854598ef95f3b2dfaac Author: Ananth Packkildurai AuthorDate: Wed Aug 11 01:34:49 2021 -0400 updated Slack community link --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 4dce9e2..61b7620 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ Apache Pinot Site = -[](https://communityinviter.com/apps/apache-pinot/apache-pinot) +[](https://join.slack.com/t/apache-pinot/shared_invite/zt-5z7pav2f-yYtjZdVA~EDmrGkho87Vzw) [](https://twitter.com/intent/follow?screen_name=apachepinot) [](LICENSE) @@ -27,7 +27,7 @@ Additional Resources Mailing list: d...@pinot.apache.org -Slack: https://communityinviter.com/apps/apache-pinot/apache-pinot +Slack: https://join.slack.com/t/apache-pinot/shared_invite/zt-5z7pav2f-yYtjZdVA~EDmrGkho87Vzw Project website: https://pinot.apache.org - To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
[pinot] branch master updated: update Slack Pinot community invite link (#7284)
This is an automated email from the ASF dual-hosted git repository. kishoreg pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/pinot.git The following commit(s) were added to refs/heads/master by this push: new d40e2d3 update Slack Pinot community invite link (#7284) d40e2d3 is described below commit d40e2d35d89252226dd56977b0fe4c08726b9a06 Author: Ananth Packkildurai AuthorDate: Wed Aug 11 01:22:58 2021 -0400 update Slack Pinot community invite link (#7284) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index f1b329d..75e8ee4 100644 --- a/README.md +++ b/README.md @@ -108,7 +108,7 @@ $ bin/quick-start-batch.sh Please refer to [Running Pinot on Kubernetes](https://docs.pinot.apache.org/basics/getting-started/kubernetes-quickstart) in our project documentation. Pinot also provides Kubernetes integrations with the interactive query engine, [Presto](kubernetes/helm/presto-coordinator.yaml), and the data visualization tool, [Apache Superset](kubernetes/helm/superset.yaml). ## Join the Community - - Ask questions on [Apache Pinot Slack](https://communityinviter.com/apps/apache-pinot/apache-pinot) + - Ask questions on [Apache Pinot Slack](https://join.slack.com/t/apache-pinot/shared_invite/zt-5z7pav2f-yYtjZdVA~EDmrGkho87Vzw) - Please join Apache Pinot mailing lists dev-subscr...@pinot.apache.org (subscribe to pinot-dev mailing list) d...@pinot.apache.org (posting to pinot-dev mailing list) - To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
[incubator-pinot] branch master updated: Fix link to superset example manifest (#6705)
This is an automated email from the ASF dual-hosted git repository. kishoreg pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git The following commit(s) were added to refs/heads/master by this push: new 593d237 Fix link to superset example manifest (#6705) 593d237 is described below commit 593d23780d0110afabebd5a2127863ee10f8fd03 Author: Tim Ebert AuthorDate: Sun Mar 21 19:20:01 2021 +0100 Fix link to superset example manifest (#6705) --- docker/images/pinot-superset/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/images/pinot-superset/README.md b/docker/images/pinot-superset/README.md index f74edfd..1237ee7 100644 --- a/docker/images/pinot-superset/README.md +++ b/docker/images/pinot-superset/README.md @@ -57,4 +57,4 @@ The data volume is located at `/app/superset_home` and it is where you would mou ## Kubernetes Examples -Please refer to [`superset.yaml`](../../../kubernetes/examples/helm/superset.yaml) as k8s deployment example. +Please refer to [`superset.yaml`](../../../kubernetes/helm/superset.yaml) as k8s deployment example. - To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
[incubator-pinot] 01/01: Fixing quickstart launcher from IDE
This is an automated email from the ASF dual-hosted git repository.
kishoreg pushed a commit to branch fix-quickstart
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
commit 6d4381a48e6cbc2603ed5b5c4685c93b11175c24
Author: kishoreg
AuthorDate: Fri Jan 29 17:56:35 2021 -0800
Fixing quickstart launcher from IDE
---
pinot-tools/pom.xml | 6 ++
1 file changed, 6 insertions(+)
diff --git a/pinot-tools/pom.xml b/pinot-tools/pom.xml
index f878ce1..43afd0f 100644
--- a/pinot-tools/pom.xml
+++ b/pinot-tools/pom.xml
@@ -103,6 +103,12 @@
org.apache.pinot
+ pinot-batch-ingestion-standalone
+ ${project.version}
+ runtime
+
+
+ org.apache.pinot
pinot-s3
${project.version}
runtime
-
To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org
For additional commands, e-mail: commits-h...@pinot.apache.org
[incubator-pinot] branch fix-quickstart created (now 6d4381a)
This is an automated email from the ASF dual-hosted git repository. kishoreg pushed a change to branch fix-quickstart in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git. at 6d4381a Fixing quickstart launcher from IDE This branch includes the following new commits: new 6d4381a Fixing quickstart launcher from IDE The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. - To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
[incubator-pinot] branch h3-index updated (e0d406f -> 35f1257)
This is an automated email from the ASF dual-hosted git repository.
kishoreg pushed a change to branch h3-index
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git.
from e0d406f Reformat H3 index config spec (#6398)
add 35f1257 Add real-time H3 index reader (#6400)
No new revisions were added by this update.
Summary of changes:
.../org/apache/pinot/core/common/DataSource.java | 2 +-
.../realtime/LLRealtimeSegmentDataManager.java | 6 +-
.../indexsegment/mutable/MutableSegmentImpl.java | 22 +--
.../operator/filter/H3IndexFilterOperator.java | 3 +-
.../core/realtime/impl/RealtimeSegmentConfig.java | 42 +---
.../impl/geospatial/RealtimeH3IndexReader.java | 76 ++
.../creator/impl/geospatial/H3IndexCreator.java| 9 ---
.../segment/index/column/ColumnIndexContainer.java | 4 +-
.../index/column/PhysicalColumnIndexContainer.java | 8 +--
.../segment/index/datasource/BaseDataSource.java | 3 +-
.../index/datasource/MutableDataSource.java| 2 +-
.../{TextIndexReader.java => H3IndexReader.java} | 21 +++---
...ndexReader.java => ImmutableH3IndexReader.java} | 8 ++-
.../virtualcolumn/VirtualColumnIndexContainer.java | 4 +-
.../impl/geospatial/RealtimeH3IndexReaderTest.java | 65 ++
.../impl/geospatial/H3IndexCreatorTest.java| 4 +-
.../meetupRsvp_realtime_table_config.json | 10 +++
.../stream/meetupRsvp/meetupRsvp_schema.json | 2 +-
18 files changed, 236 insertions(+), 55 deletions(-)
create mode 100644
pinot-core/src/main/java/org/apache/pinot/core/realtime/impl/geospatial/RealtimeH3IndexReader.java
copy
pinot-core/src/main/java/org/apache/pinot/core/segment/index/readers/{TextIndexReader.java
=> H3IndexReader.java} (68%)
rename
pinot-core/src/main/java/org/apache/pinot/core/segment/index/readers/geospatial/{H3IndexReader.java
=> ImmutableH3IndexReader.java} (94%)
create mode 100644
pinot-core/src/test/java/org/apache/pinot/core/realtime/impl/geospatial/RealtimeH3IndexReaderTest.java
-
To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org
For additional commands, e-mail: commits-h...@pinot.apache.org
[incubator-pinot] branch h3-index updated (83bc217 -> e0d406f)
This is an automated email from the ASF dual-hosted git repository.
kishoreg pushed a change to branch h3-index
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git.
from 83bc217 Cleaned up H3Index Reader/Creator (#6397)
add e0d406f Reformat H3 index config spec (#6398)
No new revisions were added by this update.
Summary of changes:
.../creator/impl/geospatial/H3IndexCreator.java| 46 ---
.../index/column/PhysicalColumnIndexContainer.java | 8 +-
.../segment/index/loader/IndexLoadingConfig.java | 23 --
.../segment/index/loader/SegmentPreProcessor.java | 2 +-
.../index/loader/invertedindex/H3IndexHandler.java | 10 +--
.../impl/geospatial/H3IndexCreatorTest.java| 92 ++
.../apache/pinot/spi/config/table/FieldConfig.java | 2 +-
.../{H3IndexConfig.java => H3IndexColumn.java} | 32 ++--
.../pinot/spi/config/table/IndexingConfig.java | 10 ---
.../starbucksStores_offline_table_config.json | 18 +++--
10 files changed, 155 insertions(+), 88 deletions(-)
create mode 100644
pinot-core/src/test/java/org/apache/pinot/core/segment/creator/impl/geospatial/H3IndexCreatorTest.java
rename
pinot-spi/src/main/java/org/apache/pinot/spi/config/table/{H3IndexConfig.java
=> H3IndexColumn.java} (57%)
-
To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org
For additional commands, e-mail: commits-h...@pinot.apache.org
[incubator-pinot] branch h3-index updated (28f59ef -> 83bc217)
This is an automated email from the ASF dual-hosted git repository.
kishoreg pushed a change to branch h3-index
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git.
from 28f59ef Added H3IndexFilterOperator (#6391)
add 83bc217 Cleaned up H3Index Reader/Creator (#6397)
No new revisions were added by this update.
Summary of changes:
.../core/common/datatable/DataTableUtils.java | 4 +-
.../indexsegment/mutable/MutableSegmentImpl.java | 4 +-
.../segment/creator/GeoSpatialIndexCreator.java| 9 +++
.../creator/impl/geospatial/H3IndexCreator.java| 89 +++---
.../segment/index/column/ColumnIndexContainer.java | 2 +-
.../index/column/PhysicalColumnIndexContainer.java | 15 ++--
.../segment/index/datasource/BaseDataSource.java | 20 +++--
.../index/datasource/ImmutableDataSource.java | 6 +-
.../index/datasource/MutableDataSource.java| 19 +++--
.../segment/index/loader/IndexLoadingConfig.java | 13 ++--
.../segment/index/loader/SegmentPreProcessor.java | 8 +-
.../index/loader/invertedindex/H3IndexHandler.java | 19 ++---
.../index/readers/geospatial/H3IndexReader.java| 29 +++
.../{UpsertConfig.java => H3IndexConfig.java} | 29 ---
.../pinot/spi/config/table/IndexingConfig.java | 11 +--
.../org/apache/pinot/tools/GenericQuickstart.java | 2 +-
.../starbucksStores_offline_table_config.json | 7 +-
17 files changed, 172 insertions(+), 114 deletions(-)
copy
pinot-spi/src/main/java/org/apache/pinot/spi/config/table/{UpsertConfig.java =>
H3IndexConfig.java} (59%)
-
To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org
For additional commands, e-mail: commits-h...@pinot.apache.org
[incubator-pinot] branch h3-index updated (4259d1c -> 28f59ef)
This is an automated email from the ASF dual-hosted git repository. kishoreg pushed a change to branch h3-index in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git. from 4259d1c revert changes on DataTypeTransformer add 28f59ef Added H3IndexFilterOperator (#6391) No new revisions were added by this update. Summary of changes: .../operator/filter/H3IndexFilterOperator.java | 72 +++--- .../org/apache/pinot/core/plan/FilterPlanNode.java | 41 +++- .../request/context/predicate/GeoPredicate.java| 72 -- .../org/apache/pinot/tools/GenericQuickstart.java | 17 +++-- .../batch/starbucksStores/rawdata/data.csv | 2 +- .../starbucksStores_offline_table_config.json | 2 + .../starbucksStores/starbucksStores_schema.json| 6 +- 7 files changed, 74 insertions(+), 138 deletions(-) delete mode 100644 pinot-core/src/main/java/org/apache/pinot/core/query/request/context/predicate/GeoPredicate.java - To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
[incubator-pinot] branch h3-index updated (561a905 -> e71f01a)
This is an automated email from the ASF dual-hosted git repository. kishoreg pushed a change to branch h3-index in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git. from 561a905 Fixing license issues add e71f01a Allow multiple H3 index resolutions (#6387) No new revisions were added by this update. Summary of changes: .../org/apache/pinot/core/common/DataSource.java | 3 +- .../transform/function/ScalarFunctions.java| 17 ++- .../operator/filter/H3IndexFilterOperator.java | 13 -- .../creator/impl/geospatial/H3IndexCreator.java| 18 +--- .../creator/impl/geospatial/H3IndexResolution.java | 53 ++ .../index/loader/invertedindex/H3IndexHandler.java | 6 ++- .../index/readers/geospatial/H3IndexReader.java| 9 +++- .../impl/geospatial/H3IndexResolutionTest.java | 17 +++ .../org/apache/pinot/tools/GenericQuickstart.java | 3 +- 9 files changed, 119 insertions(+), 20 deletions(-) create mode 100644 pinot-core/src/main/java/org/apache/pinot/core/segment/creator/impl/geospatial/H3IndexResolution.java create mode 100644 pinot-core/src/test/java/org/apache/pinot/core/segment/creator/impl/geospatial/H3IndexResolutionTest.java - To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
[incubator-pinot] branch h3-index updated (bab10b9 -> 561a905)
This is an automated email from the ASF dual-hosted git repository. kishoreg pushed a change to branch h3-index in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git. from bab10b9 Removing H3WITHIN UDF and adding logic to use h3 index for st_distance udf add 561a905 Fixing license issues No new revisions were added by this update. Summary of changes: .../transform/function/ScalarFunctions.java | 14 ++ .../transform/function/StDistanceFunction.java | 4 ++-- .../request/context/predicate/GeoPredicate.java | 18 ++ .../core/segment/creator/GeoSpatialIndexCreator.java | 18 ++ .../creator/impl/geospatial/H3IndexCreator.java | 20 .../index/readers/geospatial/H3IndexReader.java | 18 ++ .../starbucksStores/starbucksStores_schema.json | 4 ++-- 7 files changed, 92 insertions(+), 4 deletions(-) - To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
[incubator-pinot] branch h3-index updated: Removing H3WITHIN UDF and adding logic to use h3 index for st_distance udf
This is an automated email from the ASF dual-hosted git repository.
kishoreg pushed a commit to branch h3-index
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
The following commit(s) were added to refs/heads/h3-index by this push:
new bab10b9 Removing H3WITHIN UDF and adding logic to use h3 index for
st_distance udf
bab10b9 is described below
commit bab10b9536c22cce3e21a974077860e79148bb24
Author: kishoreg
AuthorDate: Mon Dec 21 09:16:46 2020 -0800
Removing H3WITHIN UDF and adding logic to use h3 index for st_distance udf
---
.../operator/filter/H3IndexFilterOperator.java | 73 +-
.../org/apache/pinot/core/plan/FilterPlanNode.java | 32 +-
.../request/context/predicate/GeoPredicate.java| 12 +---
.../creator/impl/geospatial/H3IndexCreator.java| 10 +++
.../batch/starbucksStores/rawdata/data.csv | 2 +-
5 files changed, 87 insertions(+), 42 deletions(-)
diff --git
a/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/H3IndexFilterOperator.java
b/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/H3IndexFilterOperator.java
index 7528b7e..0b65dd7 100644
---
a/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/H3IndexFilterOperator.java
+++
b/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/H3IndexFilterOperator.java
@@ -23,10 +23,19 @@ import com.uber.h3core.LengthUnit;
import java.io.IOException;
import java.util.List;
import org.apache.pinot.core.common.DataSource;
+import org.apache.pinot.core.geospatial.serde.GeometrySerializer;
+import org.apache.pinot.core.geospatial.transform.function.StPointFunction;
+import org.apache.pinot.core.indexsegment.IndexSegment;
import org.apache.pinot.core.operator.blocks.FilterBlock;
import org.apache.pinot.core.operator.docidsets.BitmapDocIdSet;
+import org.apache.pinot.core.query.request.context.ExpressionContext;
+import org.apache.pinot.core.query.request.context.FunctionContext;
import org.apache.pinot.core.query.request.context.predicate.GeoPredicate;
+import org.apache.pinot.core.query.request.context.predicate.Predicate;
+import org.apache.pinot.core.query.request.context.predicate.RangePredicate;
import org.apache.pinot.core.segment.index.readers.geospatial.H3IndexReader;
+import org.apache.pinot.spi.utils.BytesUtils;
+import org.locationtech.jts.geom.Geometry;
import org.roaringbitmap.buffer.ImmutableRoaringBitmap;
import org.roaringbitmap.buffer.MutableRoaringBitmap;
@@ -36,14 +45,51 @@ public class H3IndexFilterOperator extends
BaseFilterOperator {
// NOTE: Range index can only apply to dictionary-encoded columns for now
// TODO: Support raw index columns
- private final GeoPredicate _geoPredicate;
- private final DataSource _dataSource;
private final int _numDocs;
private final H3Core _h3Core;
+ private final H3IndexReader _h3IndexReader;
+ private Geometry _geometry;
+ private double _distance;
- public H3IndexFilterOperator(GeoPredicate geoPredicate, DataSource
dataSource, int numDocs) {
-_geoPredicate = geoPredicate;
-_dataSource = dataSource;
+ public H3IndexFilterOperator(Predicate predicate, IndexSegment indexSegment,
int numDocs) {
+FunctionContext function = predicate.getLhs().getFunction();
+String columnName;
+
+if (function.getArguments().get(0).getType() ==
ExpressionContext.Type.IDENTIFIER) {
+ columnName = function.getArguments().get(0).getIdentifier();
+ byte[] bytes =
BytesUtils.toBytes(function.getArguments().get(1).getLiteral());
+ _geometry = GeometrySerializer.deserialize(bytes);
+} else if (function.getArguments().get(1).getType() ==
ExpressionContext.Type.IDENTIFIER) {
+ columnName = function.getArguments().get(1).getIdentifier();
+ byte[] bytes =
BytesUtils.toBytes(function.getArguments().get(0).getLiteral());
+ _geometry = GeometrySerializer.deserialize(bytes);
+} else {
+ throw new RuntimeException("Expecting one of the arguments of
ST_DISTANCE to be an identifier");
+}
+DataSource dataSource = indexSegment.getDataSource(columnName);
+_h3IndexReader = dataSource.getH3Index();
+switch (predicate.getType()) {
+ case EQ:
+break;
+ case NOT_EQ:
+break;
+ case IN:
+break;
+ case NOT_IN:
+break;
+ case RANGE:
+RangePredicate rangePredicate = (RangePredicate) predicate;
+_distance = Double.parseDouble(rangePredicate.getUpperBound());
+break;
+ case REGEXP_LIKE:
+break;
+ case TEXT_MATCH:
+break;
+ case IS_NULL:
+break;
+ case IS_NOT_NULL:
+break;
+}
_numDocs = numDocs;
try {
_h3Core = H3Core.newInstance();
@@ -54,34 +100,31 @@ public class H3IndexFilterOperator extends
BaseFilterOperator {
@Override
protected FilterBlock getNextBlock() {
-H3IndexReader h3IndexReader = _dataSource.getH3Index();
//
[incubator-pinot] branch h3-index updated: Using the parameters from the query and functional
This is an automated email from the ASF dual-hosted git repository.
kishoreg pushed a commit to branch h3-index
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
The following commit(s) were added to refs/heads/h3-index by this push:
new efcec4f Using the parameters from the query and functional
efcec4f is described below
commit efcec4f7856a270a5017479643e91bc3ca9a66c5
Author: kishoreg
AuthorDate: Sun Dec 20 17:20:11 2020 -0800
Using the parameters from the query and functional
---
.../operator/filter/H3IndexFilterOperator.java | 50 +++---
.../org/apache/pinot/core/plan/FilterPlanNode.java | 12 ++
.../request/context/predicate/GeoPredicate.java| 34 ++-
.../index/readers/geospatial/H3IndexReader.java| 5 +++
4 files changed, 95 insertions(+), 6 deletions(-)
diff --git
a/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/H3IndexFilterOperator.java
b/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/H3IndexFilterOperator.java
index 13e79a7..7528b7e 100644
---
a/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/H3IndexFilterOperator.java
+++
b/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/H3IndexFilterOperator.java
@@ -18,12 +18,17 @@
*/
package org.apache.pinot.core.operator.filter;
+import com.uber.h3core.H3Core;
+import com.uber.h3core.LengthUnit;
+import java.io.IOException;
+import java.util.List;
import org.apache.pinot.core.common.DataSource;
import org.apache.pinot.core.operator.blocks.FilterBlock;
import org.apache.pinot.core.operator.docidsets.BitmapDocIdSet;
import org.apache.pinot.core.query.request.context.predicate.GeoPredicate;
import org.apache.pinot.core.segment.index.readers.geospatial.H3IndexReader;
import org.roaringbitmap.buffer.ImmutableRoaringBitmap;
+import org.roaringbitmap.buffer.MutableRoaringBitmap;
public class H3IndexFilterOperator extends BaseFilterOperator {
@@ -34,26 +39,61 @@ public class H3IndexFilterOperator extends
BaseFilterOperator {
private final GeoPredicate _geoPredicate;
private final DataSource _dataSource;
private final int _numDocs;
+ private final H3Core _h3Core;
public H3IndexFilterOperator(GeoPredicate geoPredicate, DataSource
dataSource, int numDocs) {
_geoPredicate = geoPredicate;
_dataSource = dataSource;
_numDocs = numDocs;
+try {
+ _h3Core = H3Core.newInstance();
+} catch (IOException e) {
+ throw new RuntimeException("Unable to instantiate H3", e);
//todo:log error
+}
}
@Override
protected FilterBlock getNextBlock() {
H3IndexReader h3IndexReader = _dataSource.getH3Index();
+//todo: this needs to come from somewhere?
+int resolution = 5;
+long h3Id = _h3Core
+.geoToH3(_geoPredicate.getGeometry().getCoordinate().x,
_geoPredicate.getGeometry().getCoordinate().y,
+resolution);
assert h3IndexReader != null;
-//todo: pick this from the geoPredicate
-long h3Id = h3IndexReader.getDictionary().getLongValue(0);
-ImmutableRoaringBitmap docIds = h3IndexReader.getDocIds(h3Id);
-return new FilterBlock(new BitmapDocIdSet(docIds, _numDocs) {
+
+//find the number of rings based on geopredicate.distance
+//FullMatch
+double edgeLength = _h3Core.edgeLength(resolution, LengthUnit.km);
+int numFullMatchedRings = (int) (_geoPredicate.getDistance() / edgeLength);
+List fullMatchRings = _h3Core.kRing(h3Id, numFullMatchedRings);
+fullMatchRings.add(h3Id);
+MutableRoaringBitmap fullMatchedDocIds = new MutableRoaringBitmap();
+for (long id : fullMatchRings) {
+ ImmutableRoaringBitmap docIds = h3IndexReader.getDocIds(id);
+ fullMatchedDocIds.or(docIds);
+}
+
+//partial matchedRings
+int numPartialMatchedRings = (int) (_geoPredicate.getDistance() /
edgeLength);
+List partialMatchedRings = _h3Core.kRing(h3Id,
numPartialMatchedRings);
+partialMatchedRings.add(h3Id);
+final MutableRoaringBitmap partialMatchDocIds = new MutableRoaringBitmap();
+partialMatchedRings.removeAll(fullMatchRings);
+for (long id : partialMatchedRings) {
+ ImmutableRoaringBitmap docIds = h3IndexReader.getDocIds(id);
+ partialMatchDocIds.or(docIds);
+}
+
+//TODO:evaluate the actual distance for the partial matched by scanning
+
+MutableRoaringBitmap result = ImmutableRoaringBitmap.or(fullMatchedDocIds,
partialMatchDocIds);
+return new FilterBlock(new BitmapDocIdSet(result, _numDocs) {
// Override this method to reflect the entries scanned
@Override
public long getNumEntriesScannedInFilter() {
-return 0; //TODO:Return the one from ScanBased
+return partialMatchDocIds.getCardinality();
}
});
}
diff --git
a/pinot-core/src/main/java/org/apache/pinot/core/plan/FilterPlanNode.java
b/pinot-core/src/main/java/org/apache/pinot/core/plan/FilterPlanNode.java
index f31fc7
[incubator-pinot] 02/03: Wiring H3 Index for query processing
This is an automated email from the ASF dual-hosted git repository.
kishoreg pushed a commit to branch h3-index
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
commit 5b13496533012a77efe9553a72db9cd753019b75
Author: kishoreg
AuthorDate: Sat Dec 19 09:08:32 2020 -0800
Wiring H3 Index for query processing
---
.../org/apache/pinot/core/common/DataSource.java | 9 +
.../operator/filter/H3IndexFilterOperator.java | 65 +++
.../org/apache/pinot/core/plan/FilterPlanNode.java | 17 +-
.../request/context/predicate/GeoPredicate.java| 22 +++
.../core/segment/creator/impl/V1Constants.java | 1 +
.../segment/index/column/ColumnIndexContainer.java | 6 +
.../index/column/PhysicalColumnIndexContainer.java | 15 ++
.../segment/index/datasource/BaseDataSource.java | 11 +-
.../index/datasource/ImmutableDataSource.java | 2 +-
.../index/datasource/MutableDataSource.java| 7 +-
.../segment/index/loader/IndexLoadingConfig.java | 10 +
.../segment/index/loader/SegmentPreProcessor.java | 6 +
.../index/loader/invertedindex/H3IndexHandler.java | 203 +
.../index/readers/geospatial/H3IndexReader.java| 1 -
.../pinot/core/segment/store/ColumnIndexType.java | 3 +-
.../virtualcolumn/VirtualColumnIndexContainer.java | 6 +
.../core/startree/v2/store/StarTreeDataSource.java | 3 +-
.../pinot/spi/config/table/IndexingConfig.java | 9 +
18 files changed, 385 insertions(+), 11 deletions(-)
diff --git
a/pinot-core/src/main/java/org/apache/pinot/core/common/DataSource.java
b/pinot-core/src/main/java/org/apache/pinot/core/common/DataSource.java
index 75f0513..97d3609 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/common/DataSource.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/common/DataSource.java
@@ -25,6 +25,8 @@ import
org.apache.pinot.core.segment.index.readers.ForwardIndexReader;
import org.apache.pinot.core.segment.index.readers.InvertedIndexReader;
import org.apache.pinot.core.segment.index.readers.NullValueVectorReader;
import org.apache.pinot.core.segment.index.readers.TextIndexReader;
+import org.apache.pinot.core.segment.index.readers.geospatial.H3IndexReader;
+
/**
* The {@code DataSource} contains all the indexes and metadata for a column
for query execution purpose.
@@ -61,6 +63,13 @@ public interface DataSource {
InvertedIndexReader getRangeIndex();
/**
+ * Returns the range index for the column if exists, or {@code null} if not.
+ * TODO: Have a separate interface for range index.
+ */
+ @Nullable
+ H3IndexReader getH3Index();
+
+ /**
* Returns the text index for the column if exists, or {@code null} if not.
*/
@Nullable
diff --git
a/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/H3IndexFilterOperator.java
b/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/H3IndexFilterOperator.java
new file mode 100644
index 000..98cdf64
--- /dev/null
+++
b/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/H3IndexFilterOperator.java
@@ -0,0 +1,65 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.core.operator.filter;
+
+import org.apache.pinot.core.common.DataSource;
+import org.apache.pinot.core.operator.blocks.FilterBlock;
+import org.apache.pinot.core.operator.docidsets.BitmapDocIdSet;
+import org.apache.pinot.core.query.request.context.predicate.GeoPredicate;
+import org.apache.pinot.core.segment.index.readers.geospatial.H3IndexReader;
+import org.roaringbitmap.buffer.ImmutableRoaringBitmap;
+
+
+public class H3IndexFilterOperator extends BaseFilterOperator {
+ private static final String OPERATOR_NAME = "H3IndexFilterOperator";
+
+ // NOTE: Range index can only apply to dictionary-encoded columns for now
+ // TODO: Support raw index columns
+ private final GeoPredicate _geoPredicate;
+ private final DataSource _dataSource;
+ private final int _numDocs;
+
+ public H3IndexFilterOperator(GeoPredicate geoPredicate, DataSource
dataSource, int numDocs) {
+_geoPredicate = geoPredicate;
+_dataSource = dataSource;
+_numDocs = numDocs;
+ }
+
+ @Override
+ prot
[incubator-pinot] 01/03: Initial commit for H3 based geospatial indexing
This is an automated email from the ASF dual-hosted git repository.
kishoreg pushed a commit to branch h3-index
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
commit 4bb66774f94528b6ae6607f16c8089ad4f8adfc6
Author: kishoreg
AuthorDate: Mon Nov 30 09:58:32 2020 -0800
Initial commit for H3 based geospatial indexing
---
pinot-core/pom.xml | 5 +
.../segment/creator/GeoSpatialIndexCreator.java| 9 +
.../creator/impl/geospatial/H3IndexCreator.java| 312 +
.../index/readers/BaseImmutableDictionary.java | 2 +-
.../index/readers/geospatial/H3IndexReader.java| 102 +++
5 files changed, 429 insertions(+), 1 deletion(-)
diff --git a/pinot-core/pom.xml b/pinot-core/pom.xml
index d4938a7..9a93eae 100644
--- a/pinot-core/pom.xml
+++ b/pinot-core/pom.xml
@@ -54,6 +54,11 @@
+ com.uber
+ h3
+ 3.0.3
+
+
me.lemire.integercompression
JavaFastPFOR
diff --git
a/pinot-core/src/main/java/org/apache/pinot/core/segment/creator/GeoSpatialIndexCreator.java
b/pinot-core/src/main/java/org/apache/pinot/core/segment/creator/GeoSpatialIndexCreator.java
new file mode 100644
index 000..8fcd3bd
--- /dev/null
+++
b/pinot-core/src/main/java/org/apache/pinot/core/segment/creator/GeoSpatialIndexCreator.java
@@ -0,0 +1,9 @@
+package org.apache.pinot.core.segment.creator;
+
+import java.io.Closeable;
+
+
+public interface GeoSpatialIndexCreator extends Closeable {
+
+ void add(int docId, double lat, double lon);
+}
diff --git
a/pinot-core/src/main/java/org/apache/pinot/core/segment/creator/impl/geospatial/H3IndexCreator.java
b/pinot-core/src/main/java/org/apache/pinot/core/segment/creator/impl/geospatial/H3IndexCreator.java
new file mode 100644
index 000..e2ec6a2
--- /dev/null
+++
b/pinot-core/src/main/java/org/apache/pinot/core/segment/creator/impl/geospatial/H3IndexCreator.java
@@ -0,0 +1,312 @@
+package org.apache.pinot.core.segment.creator.impl.geospatial;
+
+import com.uber.h3core.H3Core;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.ObjectInputStream;
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.PriorityQueue;
+import java.util.Random;
+import java.util.TreeMap;
+import org.apache.commons.io.FileUtils;
+import org.apache.pinot.core.segment.creator.GeoSpatialIndexCreator;
+import org.apache.pinot.core.segment.index.readers.geospatial.H3IndexReader;
+import org.apache.pinot.core.segment.memory.PinotDataBuffer;
+import org.apache.pinot.spi.data.DimensionFieldSpec;
+import org.apache.pinot.spi.data.FieldSpec;
+import org.roaringbitmap.buffer.ImmutableRoaringBitmap;
+import org.roaringbitmap.buffer.MutableRoaringBitmap;
+
+
+public class H3IndexCreator implements GeoSpatialIndexCreator {
+
+ private static final int VERSION = 1;
+ private static final int FLUSH_THRESHOLD = 100_000;
+ private final H3Core _h3Core;
+ private File _indexDir;
+ private FieldSpec _fieldSpec;
+ private int _resolution;
+
+ TreeMap _h3IndexMap;
+
+ int numChunks = 0;
+ List chunkLengths = new ArrayList<>();
+
+ public H3IndexCreator(File indexDir, FieldSpec fieldSpec, int resolution)
+ throws IOException {
+
+_indexDir = indexDir;
+_fieldSpec = fieldSpec;
+_resolution = resolution;
+_h3Core = H3Core.newInstance();
+//todo: initialize this with right size based on the
+long numHexagons = _h3Core.numHexagons(resolution);
+_h3IndexMap = new TreeMap<>();
+ }
+
+ @Override
+ public void add(int docId, double lat, double lon) {
+Long h3Id = _h3Core.geoToH3(lat, lon, _resolution);
+MutableRoaringBitmap roaringBitmap = _h3IndexMap.get(h3Id);
+if (roaringBitmap == null) {
+ roaringBitmap = new MutableRoaringBitmap();
+ _h3IndexMap.put(h3Id, roaringBitmap);
+}
+roaringBitmap.add(docId);
+if (_h3IndexMap.size() > FLUSH_THRESHOLD) {
+ flush();
+}
+ }
+
+ private void flush() {
+//dump what ever we have in _h3IndexMap in a sorted order
+try {
+
+ File tempChunkFile = new File(_indexDir, "chunk-" + numChunks);
+ DataOutputStream dos = new DataOutputStream(new
FileOutputStream(tempChunkFile));
+ chunkLengths.add(_h3IndexMap.size());
+ for (Map.Entry entry :
_h3IndexMap.entrySet()) {
+Long h3Id = entry.getKey();
+MutableRoaringBitmap bitmap = entry.getValue();
+dos.writeLong(h3Id);
+//write bitmap
+int serializedSizeInBytes = bitmap.serializedSizeInBytes();
+byte[] byteArray = new byte[serializedSizeInBytes];
+bitmap.serialize(ByteBuffer.wrap(byteArray));
+dos.writeInt(serializedSizeInBytes);
+
[incubator-pinot] branch h3-index updated (fe0e286 -> e2ea4d6)
This is an automated email from the ASF dual-hosted git repository. kishoreg pushed a change to branch h3-index in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git. discard fe0e286 Initial commit for H3 based geospatial indexing add 0dee4d1 [TE] fix changing createdTime of anomalies (#6269) add 3eb0f9c Use sorted index based filtering only for dictionary encoded column (#6288) add b31b82e [TE] frontend - harleyjj/alert-details - change alert charting logic to be agnostic of whether anomaly timestamps are included in data timestamps (#6260) add d6862a2 Adding custom metadata props into both segment metadata properties file and zk metadata record (#6299) add 5b0d1df [TE] frontend harleyjj/rca - double encode filter value to protect against special characters (#6281) add 0565f86 Make plugins to be configurable in environment variables (#6294) add 66e9394 Improve performance for distinct queries (#6285) add 3b7bfd2 [TE] frontend - rca/harleyjj - Enable forecast baseline in RCA UI (#6304) add 4be939a some geo function improvements (#6306) add 33e7ec8 [TE](feat): add formatting pipeline for js, hbs code (#6245) add 47a30ba [TE]frontend - Build new subroutes for single-metric-anomalies and composite-anomalies (#6263) add e8ceb2e [TE] rest-api - harleyjj/rca - add forecast as baseline option for metric/timeseries endpoint (#6265) add f898c18 support to add offline and realtime tables, individually able to add schema and schema listing in UI (#6296) add 9ce5d78 Include exception message in the ControllerApplicationException for /validate (#6312) add 3cf3154 [TE]frontend - Build the tree parser for composite anomalies (#6290) add 9a5cc4b Use StringUtils.replace to avoid regex for setting literal expression (#6314) add 41a7722 Fix: Close HelixAdmin appropriately, when setting up Helix Cluster. (#6315) add 8ecfd4a Update pinot_tests.yml (#6316) add 41a3fc4 API to get status of consumption of a table (#6322) add 70be687 Enhance JSONRecordReader to handle GZIP compressed JSON files. (#6321) add 682c95f Decimal percentile support. (#6323) add 22d25ff [TE]frontend - Refactor to integrate the Performance Stats API into the front-end (#6310) add fe2a63c use RoaringBitmapWriter and direct to ByteBuffer serialization in BitmapInvertedIndexCreators (#6320) add ed9f122 fixing bugs in bootstrap table tool (#6335) add a6446e9 [Controller UI] show cluster name (#6338) add c124334 Adding Pinot Minion client (#6339) add d6484f6 Added proper tooltips, ability to enable-disable table state and fixed page crash on reload status (#6327) add 7c0e22d Update .travis.yml (#6342) add 4ba7204 Fix table cache in pinot-broker (#6329) add 64d1054 Adding offline dim table creation and assignment (#6286) add 75f9fd3 Add a Controller endpoint to return table creation time (#6331) add f2c37d5 Creating a pluggable interface for Table config tuner (#6255) add e691a38 Fix the logging bug in star-tree builder (#6348) add 2796b83 Adding json path functions to extract values from json object (#6347) add 4183ffe simplify batch config and corresponding utils (#6332) add bdeec8d Fixing the issue with result schema (#6353) add 9644350 add redis module dependency (#6357) add 946ff55 [TE] fix javassist version (#6356) add fb42e72 [TE]frontend - Add breadcrumb component (#6350) add a76e766 Broker time range pruning(#6189) (#6259) add 5432099 [TE]frontend - Add a lightweight PubSub system (#6358) add 8e41708 6355 don't lose authority portion of inputDirURI (#6359) add 2ea0185 Enhance task schedule api for single type/table support (#6352) add 79b12ed Fix empty data table for distinct query (#6363) add 0522acc pinot-controller unit test suite. (#6326) add 003442a [TE]frontend - Fix the display for "NaN" values in performance-stats (#6365) add 39a24cf [TE]frontend - THIRDEYE-3772 - create a new 'composite-anomalies' component plus very simple unit integration test (#6369) add 75b0792 Fix mvn profile name for github-actions. (#6370) add 2a7d506 Improve and bug fix on json record extraction logic (#6372) new 4bb6677 Initial commit for H3 based geospatial indexing new 5b13496 Wiring H3 Index for query processing new e2ea4d6 Adding a quickstart This update added new revisions after undoing existing revisions. That is to say, some revisions that were in the old version of the branch are not in the new version. This situation occurs when a user --force pushes a change and generates a repository containing something like this: * -- * -- B -- O -- O -- O (fe0e286) \ N -- N -- N refs/heads/h3-index (e2ea4d6) You should already have received notification emails f
[incubator-pinot] 01/01: Initial commit for H3 based geospatial indexing
This is an automated email from the ASF dual-hosted git repository.
kishoreg pushed a commit to branch h3-index
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
commit fe0e2862d157373f1be98c43f06fa09a4398a7ee
Author: kishoreg
AuthorDate: Mon Nov 30 09:58:32 2020 -0800
Initial commit for H3 based geospatial indexing
---
pinot-core/pom.xml | 5 +
.../segment/creator/GeoSpatialIndexCreator.java| 9 +
.../creator/impl/geospatial/H3IndexCreator.java| 312 +
.../index/readers/BaseImmutableDictionary.java | 2 +-
.../index/readers/geospatial/H3IndexReader.java| 102 +++
5 files changed, 429 insertions(+), 1 deletion(-)
diff --git a/pinot-core/pom.xml b/pinot-core/pom.xml
index d4938a7..9a93eae 100644
--- a/pinot-core/pom.xml
+++ b/pinot-core/pom.xml
@@ -54,6 +54,11 @@
+ com.uber
+ h3
+ 3.0.3
+
+
me.lemire.integercompression
JavaFastPFOR
diff --git
a/pinot-core/src/main/java/org/apache/pinot/core/segment/creator/GeoSpatialIndexCreator.java
b/pinot-core/src/main/java/org/apache/pinot/core/segment/creator/GeoSpatialIndexCreator.java
new file mode 100644
index 000..8fcd3bd
--- /dev/null
+++
b/pinot-core/src/main/java/org/apache/pinot/core/segment/creator/GeoSpatialIndexCreator.java
@@ -0,0 +1,9 @@
+package org.apache.pinot.core.segment.creator;
+
+import java.io.Closeable;
+
+
+public interface GeoSpatialIndexCreator extends Closeable {
+
+ void add(int docId, double lat, double lon);
+}
diff --git
a/pinot-core/src/main/java/org/apache/pinot/core/segment/creator/impl/geospatial/H3IndexCreator.java
b/pinot-core/src/main/java/org/apache/pinot/core/segment/creator/impl/geospatial/H3IndexCreator.java
new file mode 100644
index 000..e2ec6a2
--- /dev/null
+++
b/pinot-core/src/main/java/org/apache/pinot/core/segment/creator/impl/geospatial/H3IndexCreator.java
@@ -0,0 +1,312 @@
+package org.apache.pinot.core.segment.creator.impl.geospatial;
+
+import com.uber.h3core.H3Core;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.ObjectInputStream;
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.PriorityQueue;
+import java.util.Random;
+import java.util.TreeMap;
+import org.apache.commons.io.FileUtils;
+import org.apache.pinot.core.segment.creator.GeoSpatialIndexCreator;
+import org.apache.pinot.core.segment.index.readers.geospatial.H3IndexReader;
+import org.apache.pinot.core.segment.memory.PinotDataBuffer;
+import org.apache.pinot.spi.data.DimensionFieldSpec;
+import org.apache.pinot.spi.data.FieldSpec;
+import org.roaringbitmap.buffer.ImmutableRoaringBitmap;
+import org.roaringbitmap.buffer.MutableRoaringBitmap;
+
+
+public class H3IndexCreator implements GeoSpatialIndexCreator {
+
+ private static final int VERSION = 1;
+ private static final int FLUSH_THRESHOLD = 100_000;
+ private final H3Core _h3Core;
+ private File _indexDir;
+ private FieldSpec _fieldSpec;
+ private int _resolution;
+
+ TreeMap _h3IndexMap;
+
+ int numChunks = 0;
+ List chunkLengths = new ArrayList<>();
+
+ public H3IndexCreator(File indexDir, FieldSpec fieldSpec, int resolution)
+ throws IOException {
+
+_indexDir = indexDir;
+_fieldSpec = fieldSpec;
+_resolution = resolution;
+_h3Core = H3Core.newInstance();
+//todo: initialize this with right size based on the
+long numHexagons = _h3Core.numHexagons(resolution);
+_h3IndexMap = new TreeMap<>();
+ }
+
+ @Override
+ public void add(int docId, double lat, double lon) {
+Long h3Id = _h3Core.geoToH3(lat, lon, _resolution);
+MutableRoaringBitmap roaringBitmap = _h3IndexMap.get(h3Id);
+if (roaringBitmap == null) {
+ roaringBitmap = new MutableRoaringBitmap();
+ _h3IndexMap.put(h3Id, roaringBitmap);
+}
+roaringBitmap.add(docId);
+if (_h3IndexMap.size() > FLUSH_THRESHOLD) {
+ flush();
+}
+ }
+
+ private void flush() {
+//dump what ever we have in _h3IndexMap in a sorted order
+try {
+
+ File tempChunkFile = new File(_indexDir, "chunk-" + numChunks);
+ DataOutputStream dos = new DataOutputStream(new
FileOutputStream(tempChunkFile));
+ chunkLengths.add(_h3IndexMap.size());
+ for (Map.Entry entry :
_h3IndexMap.entrySet()) {
+Long h3Id = entry.getKey();
+MutableRoaringBitmap bitmap = entry.getValue();
+dos.writeLong(h3Id);
+//write bitmap
+int serializedSizeInBytes = bitmap.serializedSizeInBytes();
+byte[] byteArray = new byte[serializedSizeInBytes];
+bitmap.serialize(ByteBuffer.wrap(byteArray));
+dos.writeInt(serializedSizeInBytes);
+
[incubator-pinot] branch h3-index created (now fe0e286)
This is an automated email from the ASF dual-hosted git repository. kishoreg pushed a change to branch h3-index in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git. at fe0e286 Initial commit for H3 based geospatial indexing This branch includes the following new commits: new fe0e286 Initial commit for H3 based geospatial indexing The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. - To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
[incubator-pinot] branch json-indexing updated (045f7ef -> bd6b9cc)
This is an automated email from the ASF dual-hosted git repository. kishoreg pushed a change to branch json-indexing in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git. discard 045f7ef Fixing table config discard 64dba07 adding example data for json discard e898efd changing table name from super to personJson discard e9a8ad5 Adding index creator and reader discard 8462646 Initial commit add e3b0bfc scalar functions for array (#6105) add bee125e showing Query Result as blank table and json format toggle even if query response has 0 records (#6223) add d033a11 Add profile release-sign-artifacts for pinot-spark-connector (#6229) add ca8545b [TE] frontend - abachuk/alert-details-ui-tweaks - formatting alert header (#6210) add 2d446d2 [TE] Remove maven-shade-plugin and switch from org.reflections to io.classgraph (#6238) add 4586279 [TE]frontend - Refactor detection-health and stats component (#6227) add 8ce2271 Adding support of logical functions AND and OR (#6249) add c8d7efc Make default operator for multi-term and phrase text search queries configurable (#6251) add bd2a6ce Bug-fix: Fix logging potentialy null variable. (#6252) add 750af31 Always read start/end time in millis from the segment ZK metadata (#6239) add 5a53fbe Replace sql literal regex replace (#6258) add 9eaea99 Add stream and batch to ingestionConfig (#6247) add f40c2ac Set S3 Bucket ACL policy from config (#6272) add 96647ad Allow setting HTTP headers and parameters during schema upload (#6270) add c757f24 Update download page for release 0.6.0 (#6274) add b009fd8 Improve comparison coverage for selection SQL queries in ClusterIntegrationTestUtils(#6193) (#6224) add 2a04de8 [TE] rest-api enhance implementation to calculate alert performance (#6273) add bd2905f Fix download link for 0.6.0 (#6278) add f099515 Update snapshot version to 0.7.0-SNAPSHOT (#6279) add 6d655f9 Adding config utils to apply environment variables and apply it to table config (#6271) add fa7b0e4 Perf optimization for SQL GROUP BY ORDER BY (#6225) add 4a6e094 add api for cluster manager to get table state (#6211) add c4f8ec1 update superset source repo in superset docker image build script (#6283) add 46212f4 Initial commit add ff141fa Adding index creator and reader add 8af3c11 changing table name from super to personJson add 2a4b98f adding example data for json add bd6b9cc adding support querying based on array index This update added new revisions after undoing existing revisions. That is to say, some revisions that were in the old version of the branch are not in the new version. This situation occurs when a user --force pushes a change and generates a repository containing something like this: * -- * -- B -- O -- O -- O (045f7ef) \ N -- N -- N refs/heads/json-indexing (bd6b9cc) You should already have received notification emails for all of the O revisions, and so the following emails describe only the N revisions from the common base, B. Any revisions marked "omit" are not gone; other references still refer to them. Any revisions marked "discard" are gone forever. No new revisions were added by this update. Summary of changes: docker/images/pinot-superset/Dockerfile| 5 +- docker/images/pinot-superset/README.md | 6 +- pinot-broker/pom.xml | 2 +- .../requesthandler/BaseBrokerRequestHandler.java | 2 +- .../routing/timeboundary/TimeBoundaryManager.java | 97 +++-- pinot-clients/pinot-java-client/pom.xml| 2 +- pinot-clients/pinot-jdbc-client/pom.xml| 2 +- pinot-clients/pom.xml | 2 +- pinot-common/pom.xml | 2 +- .../pinot/common/function/FunctionInvoker.java | 19 +- .../pinot/common/function/FunctionUtils.java | 32 ++ .../common/function/TransformFunctionType.java | 3 + .../common/function/scalar/ArrayFunctions.java | 80 .../pinot/common/metadata/ZKMetadataProvider.java | 5 +- .../common/metadata/segment/SegmentZKMetadata.java | 291 +++ .../apache/pinot/common/metrics/BrokerMeter.java | 3 + .../apache/pinot/common/metrics/ServerMeter.java | 2 + .../common/tier/TimeBasedTierSegmentSelector.java | 7 +- .../apache/pinot/common/utils/CommonConstants.java | 16 +- .../org/apache/pinot/common/utils/DataTable.java | 2 + .../common/utils/FileUploadDownloadClient.java | 25 +- .../apache/pinot/common/utils/PinotDataType.java | 32 +- .../common/utils/config/TableConfigUtils.java | 2 +- .../pinot/common/utils/request/RequestUtils.java | 2 +- .../pinot/common/data/DateTimeFormatSpecTest.java | 70 ++-- .../common/utils/config/TableConfigSerDeTest.java | 27
[incubator-pinot] branch json-indexing updated (64dba07 -> 045f7ef)
This is an automated email from the ASF dual-hosted git repository. kishoreg pushed a change to branch json-indexing in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git. from 64dba07 adding example data for json add 045f7ef Fixing table config No new revisions were added by this update. Summary of changes: .../examples/batch/personJson/personJson_offline_table_config.json | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) - To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
[incubator-pinot] branch json-indexing updated (9e6f837 -> 64dba07)
This is an automated email from the ASF dual-hosted git repository. kishoreg pushed a change to branch json-indexing in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git. discard 9e6f837 adding example data for json discard f20a359 changing table name from super to personJson discard cd062fb Adding index creator and reader discard 7a3062c Initial commit add b710e09 Fetch Brokers From Controller in JDBC (#5988) add 7e7a4bb Added recursive functions validation check for group by (#6186) add 05734bf Rewrite possible array aggregation functions to one level (#6127) add ef970e6 Minion command line starter for issue #6111 (#6184) add 5beb83a Reduce pql queries from 10k to 500 (#6181) add b97460c Updated README with Pinot UI GIF. (#6164) add 0b8cc88 Updating Pinot UI GIF (#6190) add ec1c859 Relaxing timeColumnName and indexingConfig validation (#6185) add 0f4ee68 Compute absolute docId in lucene collector (#6194) add 57d292c add controls for verbosity and query dialect (#6200) add 3ab13df Reduce number of sql queries from 10k to 500 in pinot-integration-tests module (#6192) add 0f85a92 add quick-start example (#6207) add 85a0804 add upsert metadata metric (#6204) add ce43288 Adding operation in table details page (#6198) add 8678f5e [Upsert] Preserve the newer added record when 2 records have the same timestamp (#6213) add cd5f812 Support running pinot batch ingestion standalone job in a parallel mode (#6214) add 413b7cb Add StrictReplicaGroupInstanceSelector (#6208) add 637b0f1 Add DateTime columns to JDBC Connection Response Metdata (#6196) add 9dd03e4 added jira reporter name between quotes (#6183) add 76eaf51 Fix the test failure caused by day time saving (#6217) add d586801 Add IN_SUBQUERY support (#6022) add 4242706 Add IN_PARTITIONED_SUBQUERY support (#6043) add aa883b8 Supporting adding of tables & schema from UI and added Timeout in Query Console (#6215) add e0f15aa Enhance TableRebalancer to support no-downtime rebalance for strict replica-group routing tables (#6212) add e5c9bec Update license and notice file for release 0.6.0 (#6219) add 75afca3 added hash functions(sha-1, sha-256, md5, etc) (#6218) add b7819e1 Adding a new Server API for computing average off heap memory consumed (#6172) add d19d604 Adding bootstrap table command and move quickstart to use it (#6220) add 9942424 Add upsert to readme (#6221) add 8462646 Initial commit add e9a8ad5 Adding index creator and reader add e898efd changing table name from super to personJson add 64dba07 adding example data for json This update added new revisions after undoing existing revisions. That is to say, some revisions that were in the old version of the branch are not in the new version. This situation occurs when a user --force pushes a change and generates a repository containing something like this: * -- * -- B -- O -- O -- O (9e6f837) \ N -- N -- N refs/heads/json-indexing (64dba07) You should already have received notification emails for all of the O revisions, and so the following emails describe only the N revisions from the common base, B. Any revisions marked "omit" are not gone; other references still refer to them. Any revisions marked "discard" are gone forever. No new revisions were added by this update. Summary of changes: LICENSE-binary | 4 +- NOTICE-binary | 12 +- README.md | 4 + .../requesthandler/BaseBrokerRequestHandler.java | 193 ++- .../pinot/broker/routing/RoutingManager.java | 90 ++-- .../instanceselector/BaseInstanceSelector.java | 87 ++-- .../routing/instanceselector/InstanceSelector.java | 18 +- .../instanceselector/InstanceSelectorFactory.java | 22 +- .../StrictReplicaGroupInstanceSelector.java| 181 +++ .../SegmentLineageBasedSegmentPreSelector.java | 3 +- .../SegmentPreSelector.java| 6 +- .../SegmentPreSelectorFactory.java | 2 +- .../segmentpruner/PartitionSegmentPruner.java | 6 +- .../routing/segmentpruner/SegmentPruner.java | 18 +- .../segmentselector/OfflineSegmentSelector.java| 7 +- .../segmentselector/RealtimeSegmentSelector.java | 7 +- .../routing/segmentselector/SegmentSelector.java | 14 +- .../routing/timeboundary/TimeBoundaryManager.java | 27 +- .../instanceselector/InstanceSelectorTest.java | 473 + .../SegmentPreSelectorTest.java| 4 +- .../routing/segmentpruner/SegmentPrunerTest.java | 11 +- .../segmentselector/SegmentSelectorTest.java | 11 +- .../timeboundary/TimeBoundaryManagerTest.java | 43 +- pinot-clients/p
[incubator-pinot] branch json-indexing updated (ee6714c -> 9e6f837)
This is an automated email from the ASF dual-hosted git repository.
kishoreg pushed a change to branch json-indexing
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git.
discard ee6714c Adding index creator and reader
discard fd32a3e Initial commit
add d2d47b2 Fix quick demo instructions #6155 (#6156)
add 2cd6abb Deep Extraction Support for ORC, Thrift, and ProtoBuf Records
(#6046)
add b22045a Do not apply metadata/dictionary based agg operator to upsert
table (#6154)
add e15fa80 Framework for adding compatibility tests (#6129)
add 0c19741 Do not prune segments for selection queries over upsert table
(#6158)
add cf675a5 Setup classpath correctly for the compat test runner (#6160)
add da451a8 Add more validation for upsert config (#6153)
add 81248d1 Update tyrus version (#6162)
add 08c46a8 Support using ordinals in GROUP BY and ORDER BY clause (#6152)
add 2484f5b add option flags for controller host:port, jar path, and
admin path (#6163)
add df4911d Bump up the helix version to 0.9.8 (#6166)
add ac7b0e7 UI integration of instance and segment operations (#6148)
add 1bf5d02 RealtimeToOfflineSegments task generator (#6124)
add 59c188d Fixing the issue of applying ordinals in order by for
distinct queries (#6171)
add d08fd5c Support reloading upsert table (#6167)
add 5817f15 Update license and notice for 0.6.0 release (#6175)
add 1c17ab8 Add table name to the log on validation failure (#6173)
add 669bd59 add execution timeout and fix exception stats (#6177)
add 73d2839 Merge common APIs for Dictionary (#6176)
add 5577e87 Add table level lock for segment upload (#6165)
add 7a3062c Initial commit
add cd062fb Adding index creator and reader
add f20a359 changing table name from super to personJson
add 9e6f837 adding example data for json
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (ee6714c)
\
N -- N -- N refs/heads/json-indexing (9e6f837)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
No new revisions were added by this update.
Summary of changes:
LICENSE-binary | 14 +-
NOTICE-binary | 14 +-
README.md |2 +-
compatibility-verifier/compCheck.sh| 45 +-
docker/images/pinot/bin/generator.sh | 79 +-
.../pinot/common/metadata/ZKMetadataProvider.java |5 +
.../common/minion/MinionTaskMetadataUtils.java | 81 +
.../RealtimeToOfflineSegmentsTaskMetadata.java | 88 +
.../pinot/common/utils/helix/HelixHelper.java |3 +-
.../apache/pinot/sql/parsers/CalciteSqlParser.java | 43 +-
.../RealtimeToOfflineSegmentsTaskMetadataTest.java | 46 +
.../pinot/sql/parsers/CalciteSqlCompilerTest.java | 70 +-
.../api/resources/PinotTableIndexingConfigs.java |3 +-
.../api/resources/PinotTableMetadataConfigs.java |3 +-
.../api/resources/PinotTableRestletResource.java | 16 +-
.../api/resources/PinotTableSegmentConfigs.java|3 +-
.../helix/core/PinotHelixResourceManager.java | 45 +-
...rInfoProvider.java => ClusterInfoAccessor.java} | 40 +-
.../helix/core/minion/PinotTaskManager.java| 10 +-
.../generator/ConvertToRawIndexTaskGenerator.java | 14 +-
.../RealtimeToOfflineSegmentsTaskGenerator.java| 312 ++
.../minion/generator/TaskGeneratorRegistry.java|7 +-
.../core/minion/generator/TaskGeneratorUtils.java | 52 +-
pinot-controller/src/main/resources/.eslintrc |7 +-
pinot-controller/src/main/resources/app/App.tsx|2 +-
.../main/resources/app/components/AppLoader.tsx|2 +-
.../main/resources/app/components/Breadcrumbs.tsx | 52 +-
.../src/main/resources/app/components/Confirm.tsx | 12 +-
.../components/{AppLoader.tsx => CustomButton.tsx} | 48 +-
.../resources/app/components/CustomCodemirror.tsx | 19 +-
.../main/resources/app/components/CustomDialog.tsx | 87 +
.../resources/app/components/CustomMultiSelect.tsx | 123 +
.../app/components/CustomNotification.tsx | 69 +
.../src/main/resources/app/components/Header.tsx |2 +-
.../app/components/Homepage/InstanceTable.tsx |6 +-
.../Homepage/Operations/EditConfigOp.tsx | 59 +
.../components/Homepage/Operations/EditTagsOp.t
[incubator-pinot] branch master updated: added jira reporter name between quotes (#6183)
This is an automated email from the ASF dual-hosted git repository.
kishoreg pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 9dd03e4 added jira reporter name between quotes (#6183)
9dd03e4 is described below
commit 9dd03e4adbb758dabda760006aa809e8754c0305
Author: cyrilou242
AuthorDate: Sun Nov 1 15:41:34 2020 +0100
added jira reporter name between quotes (#6183)
---
.../apache/pinot/thirdeye/notification/commons/ThirdEyeJiraClient.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/notification/commons/ThirdEyeJiraClient.java
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/notification/commons/ThirdEyeJiraClient.java
index 10e00d0..6e40b4d 100644
---
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/notification/commons/ThirdEyeJiraClient.java
+++
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/notification/commons/ThirdEyeJiraClient.java
@@ -104,7 +104,7 @@ public class ThirdEyeJiraClient {
StringBuilder jiraQuery = new StringBuilder();
// Query by project first as a jira optimization
jiraQuery.append("project=").append(project);
-jiraQuery.append(" and ").append("reporter IN
(").append(reporter).append(")");
+jiraQuery.append(" and ").append("reporter IN
(\"").append(reporter).append("\")");
jiraQuery.append(" and ").append(buildQueryOnLabels(labels));
jiraQuery.append(" and ").append(buildQueryOnCreatedBy(lookBackMillis));
-
To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org
For additional commands, e-mail: commits-h...@pinot.apache.org
[incubator-pinot] branch master updated: Add DateTime columns to JDBC Connection Response Metdata (#6196)
This is an automated email from the ASF dual-hosted git repository.
kishoreg pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 637b0f1 Add DateTime columns to JDBC Connection Response Metdata
(#6196)
637b0f1 is described below
commit 637b0f100b6e2886ca31205e206442216c480f7f
Author: Kartik Khare
AuthorDate: Sun Nov 1 20:10:57 2020 +0530
Add DateTime columns to JDBC Connection Response Metdata (#6196)
* Fix merge conflicts
* Use default tenant in case a tenant is not available
* Add date time columns in response
* nit: remove new lines and unused variable
---
.../main/java/org/apache/pinot/client/PinotConnectionMetaData.java | 6 ++
.../org/apache/pinot/client/controller/response/SchemaResponse.java | 3 +++
2 files changed, 9 insertions(+)
diff --git
a/pinot-clients/pinot-jdbc-client/src/main/java/org/apache/pinot/client/PinotConnectionMetaData.java
b/pinot-clients/pinot-jdbc-client/src/main/java/org/apache/pinot/client/PinotConnectionMetaData.java
index af206aa..5df5947 100644
---
a/pinot-clients/pinot-jdbc-client/src/main/java/org/apache/pinot/client/PinotConnectionMetaData.java
+++
b/pinot-clients/pinot-jdbc-client/src/main/java/org/apache/pinot/client/PinotConnectionMetaData.java
@@ -42,6 +42,7 @@ public class PinotConnectionMetaData extends
AbstractBaseConnectionMetaData {
private static final Logger LOGGER =
LoggerFactory.getLogger(PinotConnectionMetaData.class);
private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
+
private final PinotConnection _connection;
private final PinotControllerTransport _controllerTransport;
private final String _controllerURL;
@@ -169,6 +170,11 @@ public class PinotConnectionMetaData extends
AbstractBaseConnectionMetaData {
ordinalPosition++;
}
+for (JsonNode columns : schemaResponse.getDateTimeFieldSpecs()) {
+ appendColumnMeta(pinotMeta, tableName, ordinalPosition, columns);
+ ordinalPosition++;
+}
+
JsonNode resultTable = OBJECT_MAPPER.valueToTree(pinotMeta);
return PinotResultSet.fromResultTable(new
ResultTableResultSet(resultTable));
}
diff --git
a/pinot-clients/pinot-jdbc-client/src/main/java/org/apache/pinot/client/controller/response/SchemaResponse.java
b/pinot-clients/pinot-jdbc-client/src/main/java/org/apache/pinot/client/controller/response/SchemaResponse.java
index ac29348..7428bec 100644
---
a/pinot-clients/pinot-jdbc-client/src/main/java/org/apache/pinot/client/controller/response/SchemaResponse.java
+++
b/pinot-clients/pinot-jdbc-client/src/main/java/org/apache/pinot/client/controller/response/SchemaResponse.java
@@ -34,6 +34,7 @@ public class SchemaResponse {
private String _schemaName;
private JsonNode _dimensions;
private JsonNode _metrics;
+ private JsonNode _dateTimeFieldSpecs;
private SchemaResponse() {
}
@@ -42,6 +43,7 @@ public class SchemaResponse {
_schemaName = schemaResponse.get("schemaName").textValue();
_dimensions = schemaResponse.get("dimensionFieldSpecs");
_metrics = schemaResponse.get("metricFieldSpecs");
+_dateTimeFieldSpecs = schemaResponse.get("dateTimeFieldSpecs");
}
public static SchemaResponse fromJson(JsonNode schemaResponse) {
@@ -64,6 +66,7 @@ public class SchemaResponse {
return _metrics;
}
+ public JsonNode getDateTimeFieldSpecs() { return _dateTimeFieldSpecs; }
public static class SchemaResponseFuture extends
ControllerResponseFuture {
private final ObjectReader OBJECT_READER = new ObjectMapper().reader();
-
To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org
For additional commands, e-mail: commits-h...@pinot.apache.org
[incubator-pinot] branch json-indexing updated (bdfd3f1 -> ee6714c)
This is an automated email from the ASF dual-hosted git repository.
kishoreg pushed a change to branch json-indexing
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git.
discard bdfd3f1 Adding index creator and reader
new ee6714c Adding index creator and reader
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (bdfd3f1)
\
N -- N -- N refs/heads/json-indexing (ee6714c)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../antlr4/org/apache/pinot/pql/parsers/PQL2.g4| 4 +
.../pinot/common/request/AggregationInfo.java | 52 ++-
.../apache/pinot/common/request/BrokerRequest.java | 353 ++---
.../apache/pinot/common/request/DataSource.java| 11 +-
.../apache/pinot/common/request/Expression.java| 35 +-
.../pinot/common/request/ExpressionType.java | 4 +-
.../pinot/common/request/FilterOperator.java | 11 +-
.../apache/pinot/common/request/FilterQuery.java | 46 +--
.../pinot/common/request/FilterQueryMap.java | 11 +-
.../org/apache/pinot/common/request/Function.java | 21 +-
.../org/apache/pinot/common/request/GroupBy.java | 85 +++--
.../pinot/common/request/HavingFilterQuery.java| 46 +--
.../pinot/common/request/HavingFilterQueryMap.java | 11 +-
.../apache/pinot/common/request/Identifier.java| 16 +-
.../pinot/common/request/InstanceRequest.java | 76 ++---
.../org/apache/pinot/common/request/Literal.java | 4 +-
.../apache/pinot/common/request/PinotQuery.java| 61 ++--
.../apache/pinot/common/request/QuerySource.java | 11 +-
.../org/apache/pinot/common/request/QueryType.java | 31 +-
.../org/apache/pinot/common/request/Selection.java | 94 +++---
.../apache/pinot/common/request/SelectionSort.java | 16 +-
.../pinot/common/response/ProcessingException.java | 283 +++--
.../apache/pinot/parsers/utils/ParserUtils.java| 1 +
.../parsers/PinotQuery2BrokerRequestConverter.java | 1 +
.../pinot/pql/parsers/pql2/ast/FilterKind.java | 3 +-
.../apache/pinot/sql/parsers/CalciteSqlParser.java | 8 +-
pinot-common/src/thrift/request.thrift | 3 +-
.../org/apache/pinot/core/common/DataSource.java | 7 +
.../indexsegment/mutable/MutableSegmentImpl.java | 12 +-
.../operator/filter/JSONMatchFilterOperator.java | 26 +-
.../org/apache/pinot/core/plan/FilterPlanNode.java | 15 +
.../context/utils/QueryContextConverterUtils.java | 4 +
.../segment/creator/impl/inv/JSONIndexCreator.java | 12 +-
.../segment/index/column/ColumnIndexContainer.java | 7 +
.../index/column/PhysicalColumnIndexContainer.java | 15 +
.../segment/index/datasource/BaseDataSource.java | 13 +-
.../index/datasource/ImmutableDataSource.java | 5 +-
.../index/datasource/MutableDataSource.java| 5 +-
.../segment/index/loader/IndexLoadingConfig.java | 2 +-
.../loader/invertedindex/JSONIndexHandler.java | 33 +-
.../segment/index/readers/JSONIndexReader.java | 15 +-
.../virtualcolumn/VirtualColumnIndexContainer.java | 6 +
.../core/startree/v2/store/StarTreeDataSource.java | 3 +-
.../command/JSONQuickstart.java} | 146 -
44 files changed, 793 insertions(+), 831 deletions(-)
copy pinot-tools/src/main/java/org/apache/pinot/tools/{Quickstart.java =>
admin/command/JSONQuickstart.java} (54%)
-
To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org
For additional commands, e-mail: commits-h...@pinot.apache.org
[incubator-pinot] branch json-indexing created (now bdfd3f1)
This is an automated email from the ASF dual-hosted git repository. kishoreg pushed a change to branch json-indexing in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git. at bdfd3f1 Adding index creator and reader This branch includes the following new commits: new fd32a3e Initial commit new bdfd3f1 Adding index creator and reader The 2 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. - To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
[incubator-pinot] 01/02: Initial commit
This is an automated email from the ASF dual-hosted git repository.
kishoreg pushed a commit to branch json-indexing
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
commit fd32a3e2df4ee4373e95a18493473b75b823ccbf
Author: kishoreg
AuthorDate: Wed Oct 7 08:25:34 2020 -0700
Initial commit
---
pinot-core/pom.xml | 5 +
.../creator/impl/inv/NestedObjectIndexCreator.java | 158 +
2 files changed, 163 insertions(+)
diff --git a/pinot-core/pom.xml b/pinot-core/pom.xml
index c3bc4a1..4980269 100644
--- a/pinot-core/pom.xml
+++ b/pinot-core/pom.xml
@@ -164,6 +164,11 @@
json-path
+ com.github.wnameless.json
+ json-flattener
+ 0.9.0
+
+
org.locationtech.jts
jts-core
diff --git
a/pinot-core/src/main/java/org/apache/pinot/core/segment/creator/impl/inv/NestedObjectIndexCreator.java
b/pinot-core/src/main/java/org/apache/pinot/core/segment/creator/impl/inv/NestedObjectIndexCreator.java
new file mode 100644
index 000..adfa19d
--- /dev/null
+++
b/pinot-core/src/main/java/org/apache/pinot/core/segment/creator/impl/inv/NestedObjectIndexCreator.java
@@ -0,0 +1,158 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.core.segment.creator.impl.inv;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.github.wnameless.json.flattener.JsonFlattener;
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.TreeMap;
+import org.apache.pinot.spi.data.FieldSpec;
+
+
+public class NestedObjectIndexCreator {
+
+ public NestedObjectIndexCreator(File indexDir, FieldSpec fieldSpec,
FieldSpec.DataType valueType) {
+
+ }
+
+ private static List> unnestJson(JsonNode root) {
+Iterator> fields = root.fields();
+Map flattenedSingleValuesMap = new TreeMap<>();
+Map arrNodes = new TreeMap<>();
+Map objectNodes = new TreeMap<>();
+List> resultList = new ArrayList<>();
+List> tempResultList = new ArrayList<>();
+while (fields.hasNext()) {
+ Map.Entry child = fields.next();
+ if (child.getValue().isValueNode()) {
+//Normal value node
+flattenedSingleValuesMap.put(child.getKey(),
child.getValue().asText());
+ } else if (child.getValue().isArray()) {
+//Array Node: Process these nodes later
+arrNodes.put(child.getKey(), child.getValue());
+ } else {
+//Object Node
+objectNodes.put(child.getKey(), child.getValue());
+ }
+}
+for (String objectNodeKey : objectNodes.keySet()) {
+ JsonNode objectNode = objectNodes.get(objectNodeKey);
+ modifyKeysInMap(flattenedSingleValuesMap, tempResultList, objectNodeKey,
objectNode);
+}
+if (tempResultList.isEmpty()) {
+ tempResultList.add(flattenedSingleValuesMap);
+}
+if (!arrNodes.isEmpty()) {
+ for (Map flattenedMapElement : tempResultList) {
+for (String arrNodeKey : arrNodes.keySet()) {
+ JsonNode arrNode = arrNodes.get(arrNodeKey);
+ for (JsonNode arrNodeElement : arrNode) {
+modifyKeysInMap(flattenedMapElement, resultList, arrNodeKey,
arrNodeElement);
+ }
+}
+ }
+} else {
+ resultList.addAll(tempResultList);
+}
+return resultList;
+ }
+
+ private static void modifyKeysInMap(Map flattenedMap,
List> resultList,
+ String arrNodeKey, JsonNode arrNode) {
+List> objectResult = unnestJson(arrNode);
+for (Map flattenedObject : objectResult) {
+ Map flattenedObjectCopy = new TreeMap<>(flattenedMap);
+ for (Map.Entry entry : flattenedObject.entrySet()) {
+flattenedObjectCopy.put(arrNodeKey + "." + entry.getKey(),
entry.getValue());
+ }
+ resultList.add(flattenedObjectCopy);
+}
+ }
+
+ public void add(byte[] data)
+ throws IOException {
+
+JsonNode jsonNode = new ObjectMapper().readTree(data);
+
[incubator-pinot] 02/02: Adding index creator and reader
This is an automated email from the ASF dual-hosted git repository.
kishoreg pushed a commit to branch json-indexing
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
commit bdfd3f105d170f3dc02f0a25feb3b922fe3ffc6c
Author: kishoreg
AuthorDate: Sun Oct 18 22:05:51 2020 -0700
Adding index creator and reader
---
.../io/util/VarLengthBytesValueReaderWriter.java | 28 +-
.../operator/filter/BitmapBasedFilterOperator.java | 4 +-
.../core/operator/filter/FilterOperatorUtils.java | 2 +-
.../operator/filter/JSONMatchFilterOperator.java | 138 +
.../transform/function/CastTransformFunction.java | 2 +-
.../{Predicate.java => JSONMatchPredicate.java}| 57 +-
.../query/request/context/predicate/Predicate.java | 3 +-
.../core/segment/creator/impl/V1Constants.java | 1 +
.../segment/creator/impl/inv/JSONIndexCreator.java | 656 +
.../creator/impl/inv/NestedObjectIndexCreator.java | 158 -
.../segment/index/loader/IndexLoadingConfig.java | 10 +
.../segment/index/loader/SegmentPreProcessor.java | 6 +
.../loader/invertedindex/JSONIndexHandler.java | 205 +++
.../segment/index/readers/JSONIndexReader.java | 147 +
.../pinot/core/segment/store/ColumnIndexType.java | 3 +-
.../core/segment/store/FilePerIndexDirectory.java | 5 +
.../pinot/spi/config/table/IndexingConfig.java | 9 +
.../java/org/apache/pinot/spi/data/FieldSpec.java | 2 +-
18 files changed, 1249 insertions(+), 187 deletions(-)
diff --git
a/pinot-core/src/main/java/org/apache/pinot/core/io/util/VarLengthBytesValueReaderWriter.java
b/pinot-core/src/main/java/org/apache/pinot/core/io/util/VarLengthBytesValueReaderWriter.java
index 5a6a25a..88f56f6 100644
---
a/pinot-core/src/main/java/org/apache/pinot/core/io/util/VarLengthBytesValueReaderWriter.java
+++
b/pinot-core/src/main/java/org/apache/pinot/core/io/util/VarLengthBytesValueReaderWriter.java
@@ -18,6 +18,9 @@
*/
package org.apache.pinot.core.io.util;
+import java.io.ByteArrayOutputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
import java.util.Arrays;
import org.apache.pinot.common.utils.StringUtil;
import org.apache.pinot.core.segment.memory.PinotDataBuffer;
@@ -63,19 +66,19 @@ public class VarLengthBytesValueReaderWriter implements
ValueReader {
/**
* Magic bytes used to identify the dictionary files written in variable
length bytes format.
*/
- private static final byte[] MAGIC_BYTES = StringUtil.encodeUtf8(".vl;");
+ public static final byte[] MAGIC_BYTES = StringUtil.encodeUtf8(".vl;");
/**
* Increment this version if there are any structural changes in the store
format and
* deal with backward compatibility correctly based on old versions.
*/
- private static final int VERSION = 1;
+ public static final int VERSION = 1;
// Offsets of different fields in the header. Having as constants for
readability.
- private static final int VERSION_OFFSET = MAGIC_BYTES.length;
- private static final int NUM_ELEMENTS_OFFSET = VERSION_OFFSET +
Integer.BYTES;
- private static final int DATA_SECTION_OFFSET_POSITION = NUM_ELEMENTS_OFFSET
+ Integer.BYTES;
- private static final int HEADER_LENGTH = DATA_SECTION_OFFSET_POSITION +
Integer.BYTES;
+ public static final int VERSION_OFFSET = MAGIC_BYTES.length;
+ public static final int NUM_ELEMENTS_OFFSET = VERSION_OFFSET + Integer.BYTES;
+ public static final int DATA_SECTION_OFFSET_POSITION = NUM_ELEMENTS_OFFSET +
Integer.BYTES;
+ public static final int HEADER_LENGTH = DATA_SECTION_OFFSET_POSITION +
Integer.BYTES;
private final PinotDataBuffer _dataBuffer;
@@ -144,6 +147,19 @@ public class VarLengthBytesValueReaderWriter implements
ValueReader {
return false;
}
+ public static byte[] getHeaderBytes(int numElements)
+ throws IOException {
+
+ByteArrayOutputStream out = new ByteArrayOutputStream(HEADER_LENGTH);
+DataOutputStream dos = new DataOutputStream(out);
+dos.write(MAGIC_BYTES);
+dos.writeInt(VERSION);
+dos.writeInt(numElements);
+dos.writeInt(HEADER_LENGTH);
+dos.close();
+return out.toByteArray();
+ }
+
private void writeHeader() {
for (int offset = 0; offset < MAGIC_BYTES.length; offset++) {
_dataBuffer.putByte(offset, MAGIC_BYTES[offset]);
diff --git
a/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/BitmapBasedFilterOperator.java
b/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/BitmapBasedFilterOperator.java
index d9c25e1..9c307a3 100644
---
a/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/BitmapBasedFilterOperator.java
+++
b/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/BitmapBasedFilterOperator.java
@@ -38,9 +38,9 @@ public class BitmapBasedFilterOperator extends
BaseFilterOperator {
private final boolean _exclusive;
private final int _numDocs;
- BitmapBasedFilterO
[incubator-pinot] branch master updated (5577e87 -> b710e09)
This is an automated email from the ASF dual-hosted git repository.
kishoreg pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git.
from 5577e87 Add table level lock for segment upload (#6165)
add b710e09 Fetch Brokers From Controller in JDBC (#5988)
No new revisions were added by this update.
Summary of changes:
pinot-clients/pinot-jdbc-client/pom.xml| 1 -
.../org/apache/pinot/client/PinotConnection.java | 23 +++--
.../pinot/client/PinotConnectionMetaData.java | 5 +-
.../java/org/apache/pinot/client/PinotDriver.java | 12 -
.../controller/PinotControllerTransport.java | 31 ---
...se.java => ControllerTenantBrokerResponse.java} | 60 ++
.../org/apache/pinot/client/utils/DriverUtils.java | 10 +---
.../client/DummyPinotControllerTransport.java} | 31 ++-
.../apache/pinot/client/PinotConnectionTest.java | 4 +-
.../pinot/client/PinotPreparedStatementTest.java | 7 +--
.../apache/pinot/client/PinotStatementTest.java| 6 ++-
11 files changed, 109 insertions(+), 81 deletions(-)
copy
pinot-clients/pinot-jdbc-client/src/main/java/org/apache/pinot/client/controller/response/{TableResponse.java
=> ControllerTenantBrokerResponse.java} (52%)
copy
pinot-clients/{pinot-java-client/src/main/java/org/apache/pinot/client/SimpleBrokerSelector.java
=>
pinot-jdbc-client/src/test/java/org/apache/pinot/client/DummyPinotControllerTransport.java}
(50%)
-
To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org
For additional commands, e-mail: commits-h...@pinot.apache.org
[incubator-pinot] branch master updated: Adding Tenants, Instances, Tables, Segments count tiles and their respective pages (#6117)
This is an automated email from the ASF dual-hosted git repository.
kishoreg pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 02dd3e2 Adding Tenants, Instances, Tables, Segments count tiles and
their respective pages (#6117)
02dd3e2 is described below
commit 02dd3e2f9dcb9df79f911fbe010a2d5e9db4721a
Author: Sanket Shah
AuthorDate: Thu Oct 8 00:22:44 2020 +0530
Adding Tenants, Instances, Tables, Segments count tiles and their
respective pages (#6117)
* Adding Tenants, Instances, Tables, Segments count tiles and their
respective pages
* removed segment count from homepage
---
.../main/resources/app/components/Breadcrumbs.tsx | 5 +
.../src/main/resources/app/components/Header.tsx | 4 +-
.../app/components/Homepage/InstanceTable.tsx | 2 +-
.../app/components/Homepage/InstancesTables.tsx| 34 +-
.../{TenantsTable.tsx => TenantsListing.tsx} | 26 +
.../src/main/resources/app/components/Table.tsx| 34 +++---
.../src/main/resources/app/interfaces/types.d.ts | 13 ++-
.../src/main/resources/app/pages/HomePage.tsx | 126 +++--
.../InstanceListingPage.tsx} | 54 +
.../src/main/resources/app/pages/Query.tsx | 2 +-
.../pages/{Tenants.tsx => TablesListingPage.tsx} | 65 +++
.../src/main/resources/app/pages/Tenants.tsx | 40 ++-
.../TenantsListingPage.tsx}| 50
.../src/main/resources/app/requests/index.ts | 15 ++-
pinot-controller/src/main/resources/app/router.tsx | 8 ++
.../main/resources/app/utils/PinotMethodUtils.ts | 74
16 files changed, 381 insertions(+), 171 deletions(-)
diff --git a/pinot-controller/src/main/resources/app/components/Breadcrumbs.tsx
b/pinot-controller/src/main/resources/app/components/Breadcrumbs.tsx
index 3249af6..b77104f 100644
--- a/pinot-controller/src/main/resources/app/components/Breadcrumbs.tsx
+++ b/pinot-controller/src/main/resources/app/components/Breadcrumbs.tsx
@@ -47,6 +47,11 @@ const LinkRouter = (props: LinkRouterProps) => (
const breadcrumbNameMap: { [key: string]: string } = {
'/': 'Home',
+ '/tenants': 'Tenants',
+ '/controllers': 'Controllers',
+ '/brokers': 'Brokers',
+ '/servers': 'Servers',
+ '/tables': 'Tables',
'/query': 'Query Console',
'/cluster': 'Cluster Manager',
'/zookeeper': 'Zookeeper Browser'
diff --git a/pinot-controller/src/main/resources/app/components/Header.tsx
b/pinot-controller/src/main/resources/app/components/Header.tsx
index 08817b0..89532a8 100644
--- a/pinot-controller/src/main/resources/app/components/Header.tsx
+++ b/pinot-controller/src/main/resources/app/components/Header.tsx
@@ -34,10 +34,10 @@ const Header = ({ highlightSidebarLink,
showHideSideBarHandler, openSidebar, ...
- highlightSidebarLink(1)}
fulllogo={openSidebar.toString()} />
+
highlightSidebarLink(1)} fulllogo={openSidebar.toString()} />
-
+
showHideSideBarHandler()} />
diff --git
a/pinot-controller/src/main/resources/app/components/Homepage/InstanceTable.tsx
b/pinot-controller/src/main/resources/app/components/Homepage/InstanceTable.tsx
index 3e2da1c..60b9018 100644
---
a/pinot-controller/src/main/resources/app/components/Homepage/InstanceTable.tsx
+++
b/pinot-controller/src/main/resources/app/components/Homepage/InstanceTable.tsx
@@ -25,7 +25,7 @@ import PinotMethodUtils from '../../utils/PinotMethodUtils';
type Props = {
name: string,
- instances: string[],
+ instances: Array,
clusterName: string
};
diff --git
a/pinot-controller/src/main/resources/app/components/Homepage/InstancesTables.tsx
b/pinot-controller/src/main/resources/app/components/Homepage/InstancesTables.tsx
index bca24ca..6f09121 100644
---
a/pinot-controller/src/main/resources/app/components/Homepage/InstancesTables.tsx
+++
b/pinot-controller/src/main/resources/app/components/Homepage/InstancesTables.tsx
@@ -17,42 +17,16 @@
* under the License.
*/
-import React, { useEffect, useState } from 'react';
+import React, { } from 'react';
import map from 'lodash/map';
-import AppLoader from '../AppLoader';
import InstanceTable from './InstanceTable';
-import PinotMethodUtils from '../../utils/PinotMethodUtils';
-type DataTable = {
- [name: string]: string[]
-};
-
-const Instances = () => {
- const [fetching, setFetching] = useState(true);
- const [instances, setInstances] = useState();
- const [clusterName, setClusterName] = u
[incubator-pinot] branch master updated: Add support for Decimal with Precision Sum aggregation (#6053)
This is an automated email from the ASF dual-hosted git repository.
kishoreg pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
The following commit(s) were added to refs/heads/master by this push:
new deb3891 Add support for Decimal with Precision Sum aggregation (#6053)
deb3891 is described below
commit deb389182209db4a18761be9e20d7dcbf037b16b
Author: Kartik Khare
AuthorDate: Fri Oct 2 00:57:32 2020 +0530
Add support for Decimal with Precision Sum aggregation (#6053)
* Add support for big decimal
* Add transform function to factory
* Add support for decimal with precision addition
* Add license header
* Add Sum Precision aggregation function
* Remove add with precision transform function
* Add license header
* Refactor: Correct import of Scalar function
* Add function to convert normal string to bigdecimal bytes
* Add test for big decimal
* Add test for big decimal precision
* Add support for scale along with precision
* Add license header
* Add base64 encode functions
* typo fix
* Move arguments logic inside constructor
* set scale and precision only in final result
* Reduce scale bytes from 4 to 2
* Add java docs for sum with precision function
* Rename sumWithPrecision to sumPrecision
* Adding methods to directly take big decimal input
Co-authored-by: Kartik Khare
---
.../common/function/AggregationFunctionType.java | 1 +
.../scalar/DataTypeConversionFunctions.java| 142 +
.../apache/pinot/core/common/ObjectSerDeUtils.java | 31 ++-
.../function/AggregationFunctionFactory.java | 2 +
.../function/SumPrecisionAggregationFunction.java | 180 +
.../function/AggregationFunctionFactoryTest.java | 7 +
.../apache/pinot/queries/SumWithPrecisionTest.java | 221 +
7 files changed, 581 insertions(+), 3 deletions(-)
diff --git
a/pinot-common/src/main/java/org/apache/pinot/common/function/AggregationFunctionType.java
b/pinot-common/src/main/java/org/apache/pinot/common/function/AggregationFunctionType.java
index aeae907..37517b2 100644
---
a/pinot-common/src/main/java/org/apache/pinot/common/function/AggregationFunctionType.java
+++
b/pinot-common/src/main/java/org/apache/pinot/common/function/AggregationFunctionType.java
@@ -30,6 +30,7 @@ public enum AggregationFunctionType {
MIN("min"),
MAX("max"),
SUM("sum"),
+ SUMPRECISION("sumPrecision"),
AVG("avg"),
MINMAXRANGE("minMaxRange"),
DISTINCTCOUNT("distinctCount"),
diff --git
a/pinot-common/src/main/java/org/apache/pinot/common/function/scalar/DataTypeConversionFunctions.java
b/pinot-common/src/main/java/org/apache/pinot/common/function/scalar/DataTypeConversionFunctions.java
new file mode 100644
index 000..d9ec3b1
--- /dev/null
+++
b/pinot-common/src/main/java/org/apache/pinot/common/function/scalar/DataTypeConversionFunctions.java
@@ -0,0 +1,142 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.common.function.scalar;
+
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.util.Base64;
+import org.apache.pinot.spi.annotations.ScalarFunction;
+
+
+/**
+ * Contains function to convert a datatype to another datatype.
+ */
+public class DataTypeConversionFunctions {
+ private DataTypeConversionFunctions() {
+
+ }
+
+ /**
+ * Converts big decimal string representation to bytes.
+ * Only scale of upto 2 bytes is supported by the function
+ * @param number big decimal number in plain string. e.g. '1234.12121'
+ * @return The result byte array contains the bytes of the unscaled value
appended to bytes of the scale in BIG ENDIAN order.
+ */
+ @ScalarFunction
+ public static byte[] bigDecimalToBytes(String number) {
+BigDecimal bigDecimal = new BigDecimal(number);
+return bigDecimalToBytes(bigDecimal);
+ }
+
+ /**
+ * Converts bytes value representation generated by {
[incubator-pinot] branch master updated: Support for Update & Delete in ZooKeeper Browser and added SQL Functions in SQL Editor autocomplete list (#5981)
This is an automated email from the ASF dual-hosted git repository.
kishoreg pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 5da3433 Support for Update & Delete in ZooKeeper Browser and added
SQL Functions in SQL Editor autocomplete list (#5981)
5da3433 is described below
commit 5da34330bce20ea1f4b86b47b2b2266a764e29c1
Author: Sanket Shah
AuthorDate: Tue Sep 15 19:38:16 2020 +0530
Support for Update & Delete in ZooKeeper Browser and added SQL Functions in
SQL Editor autocomplete list (#5981)
* Adding api to edit ZK path
* Adding delete api
* Support for Update & Delete in ZooKeeper Browser and added SQL Functions
in SQL Editor autocomplete list
* showing notification on operation completion, display last refresh time,
fixed refresh action
Co-authored-by: kishoreg
---
.../src/main/resources/app/components/Confirm.tsx | 106
.../resources/app/components/CustomCodemirror.tsx | 66 ++
.../app/components/Zookeeper/TreeDirectory.tsx | 136 +++--
.../src/main/resources/app/interfaces/types.d.ts | 1 +
.../src/main/resources/app/pages/Query.tsx | 9 ++
.../src/main/resources/app/pages/ZookeeperPage.tsx | 63 +-
.../src/main/resources/app/requests/index.ts | 10 +-
.../src/main/resources/app/styles/styles.css | 5 +
.../main/resources/app/utils/PinotMethodUtils.ts | 22 +++-
.../src/main/resources/app/utils/Utils.tsx | 31 ++---
.../src/main/resources/app/utils/axios-config.ts | 2 +-
11 files changed, 394 insertions(+), 57 deletions(-)
diff --git a/pinot-controller/src/main/resources/app/components/Confirm.tsx
b/pinot-controller/src/main/resources/app/components/Confirm.tsx
new file mode 100644
index 000..bb11f99
--- /dev/null
+++ b/pinot-controller/src/main/resources/app/components/Confirm.tsx
@@ -0,0 +1,106 @@
+/* eslint-disable no-nested-ternary */
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import React, { useEffect } from 'react';
+import { Button, Dialog, DialogTitle, DialogContent, DialogContentText,
DialogActions, makeStyles } from '@material-ui/core';
+import { green, red } from '@material-ui/core/colors';
+
+const useStyles = makeStyles((theme) => ({
+ dialogContent: {
+minWidth: 900
+ },
+ dialogTextContent: {
+fontWeight: 600
+ },
+ dialogActions: {
+justifyContent: 'center'
+ },
+ green: {
+fontWeight: 600,
+color: green[500],
+borderColor: green[500],
+'&:hover': {
+ backgroundColor: green[50],
+ borderColor: green[500]
+}
+ },
+ red: {
+fontWeight: 600,
+color: red[500],
+borderColor: red[500],
+'&:hover': {
+ backgroundColor: red[50],
+ borderColor: red[500]
+}
+ }
+}));
+
+
+type Props = {
+ openDialog: boolean,
+ dialogTitle?: string,
+ dialogContent: string,
+ successCallback: (event: React.MouseEvent) =>
void;
+ closeDialog: (event: React.MouseEvent) =>
void;
+ dialogYesLabel?: string,
+ dialogNoLabel?: string
+};
+
+const Confirm = ({openDialog, dialogTitle, dialogContent, successCallback,
closeDialog, dialogYesLabel, dialogNoLabel}: Props) => {
+ const classes = useStyles();
+ const [open, setOpen] = React.useState(openDialog);
+
+ useEffect(()=>{
+setOpen(openDialog);
+ }, [openDialog])
+
+ const isStringDialog = typeof dialogContent === 'string';
+
+ return (
+
+
+{dialogTitle && {dialogTitle}}
+
+ {isStringDialog ?
+
+ {dialogContent}
+
+ : dialogContent}
+
+
+
+{dialogNoLabel || "No"}
+
+
+{dialogYesLabel || "Yes"}
+
+
+
+
+ );
+};
+
+export default Confirm;
\ No newline at end of file
diff --git
a/pinot-controller/src/main/resources/app/components/CustomCodemirror.tsx
b/pinot-controller/s
[incubator-pinot] branch master updated: Feature/#5390 segment indexing reload status api (#5718)
This is an automated email from the ASF dual-hosted git repository. kishoreg pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git The following commit(s) were added to refs/heads/master by this push: new dc77271 Feature/#5390 segment indexing reload status api (#5718) dc77271 is described below commit dc772712f3d52c0c91caa774c320f9ca54b855dd Author: Guruguha AuthorDate: Mon Sep 14 09:28:04 2020 -0700 Feature/#5390 segment indexing reload status api (#5718) * - initial feature push * Segment Reload API - added a new API endpoint for users to query segment reload status API - Table metadata from Server - added a new endpoint to fetch segment metadata - added helper classes and methods to fetch metadata from the server Tests - added test to server API to fetch metadata including indexing information * Code Refactor: - Moved status classes to logical places Logs - Added logging statements Tests - Added unit tests for Pinot Controller reload status and segment metadata API - Added unit tests for Pinot Server reload status and segment metadata API License Headers - Add license headers to files added to this feature * Code Review Changes - Updating code as per PR review comments * Adding comments to new classes and methods added as part of this feature Removing SegmentMetadataFetcher as it seemed redundant Refactoring code to save failed segment reload status API calls as part of response * Code refactor to accommodate PR comments Pinot codestyle corrections Moving ServerSegmentMetadataReader to util * Updating API definition for loadStatus * Code refactor to remove duplicate code * Code refactor to remove duplicate code * Code refactor as per PR comments * Updating segment loadStatus API to return long time than readable string * - Bug fix on the server API endpoint * Adding pretty print for server metadata response * Reverting incorrect filename refactoring Updating variable names to reflect their value type * removing unused method variable * Enabling Pretty print of server response. Code refactor to clean up lines that went beyond line length * Removing reload status API * Pretty print result * Fix test Co-authored-by: Neha Pawar --- .../api/resources/PinotSegmentRestletResource.java | 59 +- .../api/resources/ServerTableSizeReader.java | 49 ++--- .../controller/util/CompletionServiceHelper.java | 108 ++ .../util/ServerSegmentMetadataReader.java | 92 + .../pinot/controller/util/TableMetadataReader.java | 74 +++ .../controller/api/PinotSegmentsMetadataTest.java | 230 + .../immutable/ImmutableSegmentImpl.java| 4 + .../api/resources/SegmentMetadataFetcher.java | 135 .../pinot/server/api/resources/TablesResource.java | 22 +- .../pinot/server/api/TablesResourceTest.java | 55 - 10 files changed, 773 insertions(+), 55 deletions(-) diff --git a/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotSegmentRestletResource.java b/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotSegmentRestletResource.java index cd622d2..876d3a9 100644 --- a/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotSegmentRestletResource.java +++ b/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotSegmentRestletResource.java @@ -19,14 +19,17 @@ package org.apache.pinot.controller.api.resources; import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; import io.swagger.annotations.Api; import io.swagger.annotations.ApiOperation; import io.swagger.annotations.ApiParam; +import java.io.IOException; import java.util.ArrayList; import java.util.Collections; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.concurrent.Executor; import javax.annotation.Nullable; import javax.inject.Inject; import javax.ws.rs.Consumes; @@ -42,16 +45,20 @@ import javax.ws.rs.WebApplicationException; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; import javax.ws.rs.core.Response.Status; +import org.apache.commons.httpclient.HttpConnectionManager; import org.apache.helix.ZNRecord; import org.apache.helix.store.zk.ZkHelixPropertyStore; +import org.apache.pinot.common.exception.InvalidConfigException; import org.apache.pinot.common.exception.TableNotFoundException; import org.apache.pinot.common.metadata.ZKMetadataProvider; import org.apache.pinot.common.metadata.segment.OfflineSegmentZKMetadata; import
[incubator-pinot] branch master updated: Zookeeper put api (#5949)
This is an automated email from the ASF dual-hosted git repository.
kishoreg pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 83598ce Zookeeper put api (#5949)
83598ce is described below
commit 83598ceca137da00087f728338d1de5fc96ced9d
Author: Kishore Gopalakrishna
AuthorDate: Sun Sep 13 11:34:54 2020 -0700
Zookeeper put api (#5949)
* Adding api to edit ZK path
* Adding delete api
* Addressing comments
---
.../api/resources/ZookeeperResource.java | 62 ++
.../helix/core/PinotHelixResourceManager.java | 17 --
2 files changed, 75 insertions(+), 4 deletions(-)
diff --git
a/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/ZookeeperResource.java
b/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/ZookeeperResource.java
index 3093052..d367f2c 100644
---
a/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/ZookeeperResource.java
+++
b/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/ZookeeperResource.java
@@ -19,6 +19,7 @@
package org.apache.pinot.controller.api.resources;
import com.fasterxml.jackson.core.JsonProcessingException;
+import com.google.common.base.Charsets;
import io.swagger.annotations.Api;
import io.swagger.annotations.ApiOperation;
import io.swagger.annotations.ApiParam;
@@ -28,13 +29,17 @@ import java.nio.charset.StandardCharsets;
import java.util.List;
import java.util.Map;
import javax.inject.Inject;
+import javax.ws.rs.DELETE;
import javax.ws.rs.DefaultValue;
import javax.ws.rs.GET;
+import javax.ws.rs.POST;
+import javax.ws.rs.PUT;
import javax.ws.rs.Path;
import javax.ws.rs.Produces;
import javax.ws.rs.QueryParam;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
+import org.apache.helix.AccessOption;
import org.apache.helix.ZNRecord;
import org.apache.helix.manager.zk.ZNRecordSerializer;
import org.apache.pinot.controller.helix.core.PinotHelixResourceManager;
@@ -75,6 +80,63 @@ public class ZookeeperResource {
return null;
}
+ @DELETE
+ @Path("/zk/delete")
+ @Produces(MediaType.APPLICATION_JSON)
+ @ApiOperation(value = "Delete the znode at this path")
+ @ApiResponses(value = { //
+ @ApiResponse(code = 200, message = "Success"), //
+ @ApiResponse(code = 404, message = "ZK Path not found"), //
+ @ApiResponse(code = 204, message = "No Content"), //
+ @ApiResponse(code = 500, message = "Internal server error")})
+ public SuccessResponse delete(
+ @ApiParam(value = "Zookeeper Path, must start with /", required = true,
defaultValue = "/") @QueryParam("path") @DefaultValue("") String path) {
+
+path = validateAndNormalizeZKPath(path);
+
+boolean success = pinotHelixResourceManager.deleteZKPath(path);
+if (success) {
+ return new SuccessResponse("Successfully deleted path: " + path);
+} else {
+ throw new ControllerApplicationException(LOGGER, "Failed to delete path:
" + path,
+ Response.Status.INTERNAL_SERVER_ERROR);
+}
+ }
+
+ @PUT
+ @Path("/zk/put")
+ @Produces(MediaType.APPLICATION_JSON)
+ @ApiOperation(value = "Update the content of the node")
+ @ApiResponses(value = { //
+ @ApiResponse(code = 200, message = "Success"), //
+ @ApiResponse(code = 404, message = "ZK Path not found"), //
+ @ApiResponse(code = 204, message = "No Content"), //
+ @ApiResponse(code = 500, message = "Internal server error")})
+ public SuccessResponse putData(
+ @ApiParam(value = "Zookeeper Path, must start with /", required = true,
defaultValue = "/") @QueryParam("path") @DefaultValue("") String path,
+ @ApiParam(value = "Content", required = true) @QueryParam("data")
@DefaultValue("") String content,
+ @ApiParam(value = "expectedVersion", required = true, defaultValue =
"-1") @QueryParam("expectedVersion") @DefaultValue("-1") String expectedVersion,
+ @ApiParam(value = "accessOption", required = true, defaultValue = "1")
@QueryParam("accessOption") @DefaultValue("1") String accessOption) {
+path = validateAndNormalizeZKPath(path);
+ZNRecord record = null;
+if (content != null) {
+ record = (ZNRecord)
_znRecordSerializer.deserialize(content.getBytes(Charsets.UTF_8));
+}
+try {
+ boolean result = pinotHelixResourceManager
+ .setZKData(path, record, Integer.parseInt(expectedVersion),
Integer.parseInt(accessOption));
+ if (result) {
+return new SuccessRespons
[incubator-pinot] branch zookeeper-put-api updated (d3e86ad -> 3c2c88a)
This is an automated email from the ASF dual-hosted git repository. kishoreg pushed a change to branch zookeeper-put-api in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git. from d3e86ad Adding delete api add 3c2c88a Addressing comments No new revisions were added by this update. Summary of changes: .../pinot/controller/api/resources/ZookeeperResource.java | 10 +- 1 file changed, 5 insertions(+), 5 deletions(-) - To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
[incubator-pinot] branch master updated (e0ed179 -> 0c1d604)
This is an automated email from the ASF dual-hosted git repository. kishoreg pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git. from e0ed179 Validate timeColumnName when adding/updating schema/tableConfig (#5966) add 0c1d604 [TE] Moved interfaces and datalayer to a new thirdeye-spi module. (#5991) No new revisions were added by this update. Summary of changes: thirdeye/pom.xml | 20 +++- thirdeye/thirdeye-pinot/pom.xml| 46 + .../thirdeye/alert/feed/UnionAnomalyFeed.java | 51 +- .../thirdeye/alert/fetcher/BaseAnomalyFetcher.java | 5 + .../alert/fetcher/ContinuumAnomalyFetcher.java | 2 +- .../alert/fetcher/UnnotifiedAnomalyFetcher.java| 2 +- .../AnomalyDetectionInputContextBuilder.java | 3 +- .../dashboard/resources/v2/AnomaliesResource.java | 20 ++-- .../thirdeye/dataframe/util/DataFrameUtils.java| 16 +--- .../thirdeye/datalayer/bao/EvaluationManager.java | 33 --- .../datalayer/bao/OnlineDetectionDataManager.java | 9 -- .../datalayer/dto/OnlineDetectionDataDTO.java | 6 -- .../datasource/loader/AggregationLoader.java | 3 +- .../datasource/loader/TimeSeriesLoader.java| 5 +- .../datasource/mock/MockThirdEyeDataSource.java| 4 +- .../resultset/ThirdEyeDataFrameResultSet.java | 4 +- .../detection/CurrentAndBaselineLoader.java| 7 +- .../thirdeye/detection/DefaultDataProvider.java| 7 +- .../thirdeye/detection/DetectionPipeline.java | 6 +- .../thirdeye/detection/DetectionResource.java | 10 +- .../pinot/thirdeye/detection/DetectionUtils.java | 15 ++- .../detection/algorithm/AlgorithmUtils.java| 5 +- .../detection/algorithm/DimensionWrapper.java | 23 ++--- .../algorithm/LegacyAnomalyFunctionAlgorithm.java | 10 +- .../algorithm/LegacyDimensionWrapper.java | 5 +- .../detection/algorithm/LegacyMergeWrapper.java| 17 ++-- .../cache/builder/TimeSeriesCacheBuilder.java | 10 +- .../AbsoluteChangeRuleAnomalyFilter.java | 5 +- .../components/AbsoluteChangeRuleDetector.java | 22 ++--- .../detection/components/HoltWintersDetector.java | 54 +-- .../components/MeanVarianceRuleDetector.java | 41 .../PercentageChangeRuleAnomalyFilter.java | 4 +- .../components/PercentageChangeRuleDetector.java | 24 ++--- .../SitewideImpactRuleAnomalyFilter.java | 8 +- .../components/ThresholdRuleDetector.java | 20 ++-- .../thirdeye/detection/health/HealthStatus.java| 28 -- .../thirdeye/detection/spi/model/TimeSeries.java | 50 +- .../detection/wrapper/AnomalyDetectorWrapper.java | 9 +- .../thirdeye/detection/wrapper/GrouperWrapper.java | 4 +- .../thirdeye/detection/yaml/YamlResource.java | 18 ++-- .../rootcause/callgraph/CallGraphPipeline.java | 4 +- .../rootcause/impl/DimensionAnalysisPipeline.java | 2 +- .../rootcause/impl/MetricAnalysisPipeline.java | 4 +- .../rootcause/impl/MetricAnalysisPipeline2.java| 4 +- .../rootcause/impl/MetricBreakdownPipeline.java| 2 +- .../impl/MetricComponentAnalysisPipeline.java | 6 +- .../impl/MetricCorrelationRankingPipeline.java | 4 +- .../thirdeye/rootcause/timeseries/Baseline.java| 5 +- .../apache/pinot/thirdeye/util/ThirdEyeUtils.java | 52 +- .../csv/CSVThirdEyeDataSourceIntegrationTest.java | 2 +- .../MockThirdEyeDataSourceIntegrationTest.java | 20 ++-- .../pinot/thirdeye/detection/DataProviderTest.java | 7 +- .../detection/DefaultInputDataFetcherTest.java | 6 +- .../pinot/thirdeye/detection/MockDataProvider.java | 13 +-- .../detection/algorithm/AlgorithmUtilsTest.java| 22 ++--- .../detection/algorithm/DimensionWrapperTest.java | 4 +- .../AbsoluteChangeRuleAnomalyFilterTest.java | 15 +-- .../components/AbsoluteChangeRuleDetectorTest.java | 9 +- .../components/HoltWintersDetectorTest.java| 9 +- .../components/MeanVarianceRuleDetectorTest.java | 6 +- .../detection/components/MockGrouperTest.java | 21 +++-- .../PercentageChangeRuleAnomalyFilterTest.java | 21 +++-- .../PercentageChangeRuleDetectorTest.java | 22 ++--- .../components/RuleBaselineProviderTest.java | 17 ++-- .../SitewideImpactRuleAnomalyFilterTest.java | 15 +-- .../components/ThresholdRuleAnomalyFilterTest.java | 15 +-- .../components/ThresholdRuleDetectorTest.java | 9 +- .../components/ThresholdSeverityLabelerTest.java | 9 +- .../dataquality/DataQualityTaskRunnerTest.java | 9 +- .../wrapper/AnomalyDetectorWrapperTest.java| 11 +-- .../wrapper/BaselineFillingMergeWrapperTest.java | 8 +- thirdeye/thirdeye-spi/pom.xml | 105 + .../thirdeye/alert/commons/AnomalyFeedConfig.java
[incubator-pinot] branch master updated: Adding a null check inside getDataSource method for potentially invalid column name (#5923)
This is an automated email from the ASF dual-hosted git repository.
kishoreg pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
The following commit(s) were added to refs/heads/master by this push:
new cd86f60 Adding a null check inside getDataSource method for
potentially invalid column name (#5923)
cd86f60 is described below
commit cd86f60f92268f1408b79423bfe6c8eccda1ba07
Author: icefury71
AuthorDate: Wed Sep 9 11:51:46 2020 -0700
Adding a null check inside getDataSource method for potentially invalid
column name (#5923)
---
.../pinot/core/indexsegment/immutable/ImmutableSegmentImpl.java | 8 +++-
.../pinot/core/indexsegment/mutable/MutableSegmentImpl.java | 3 ++-
2 files changed, 9 insertions(+), 2 deletions(-)
diff --git
a/pinot-core/src/main/java/org/apache/pinot/core/indexsegment/immutable/ImmutableSegmentImpl.java
b/pinot-core/src/main/java/org/apache/pinot/core/indexsegment/immutable/ImmutableSegmentImpl.java
index 42484cb..52537f5 100644
---
a/pinot-core/src/main/java/org/apache/pinot/core/indexsegment/immutable/ImmutableSegmentImpl.java
+++
b/pinot-core/src/main/java/org/apache/pinot/core/indexsegment/immutable/ImmutableSegmentImpl.java
@@ -23,9 +23,12 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import javax.annotation.Nullable;
+
+import com.google.common.base.Preconditions;
import org.apache.pinot.core.common.DataSource;
import org.apache.pinot.core.segment.index.column.ColumnIndexContainer;
import org.apache.pinot.core.segment.index.datasource.ImmutableDataSource;
+import org.apache.pinot.core.segment.index.metadata.ColumnMetadata;
import org.apache.pinot.core.segment.index.metadata.SegmentMetadataImpl;
import org.apache.pinot.core.segment.index.readers.Dictionary;
import org.apache.pinot.core.segment.index.readers.ForwardIndexReader;
@@ -91,7 +94,10 @@ public class ImmutableSegmentImpl implements
ImmutableSegment {
@Override
public DataSource getDataSource(String column) {
-return new
ImmutableDataSource(_segmentMetadata.getColumnMetadataFor(column),
_indexContainerMap.get(column));
+ColumnMetadata columnMetadata =
_segmentMetadata.getColumnMetadataFor(column);
+Preconditions.checkNotNull(columnMetadata,
+"ColumnMetadata for " + column + " should not be null. " +
"Potentially invalid column name specified.");
+return new ImmutableDataSource(columnMetadata,
_indexContainerMap.get(column));
}
@Override
diff --git
a/pinot-core/src/main/java/org/apache/pinot/core/indexsegment/mutable/MutableSegmentImpl.java
b/pinot-core/src/main/java/org/apache/pinot/core/indexsegment/mutable/MutableSegmentImpl.java
index 5f04f0e..d6cfcc6 100644
---
a/pinot-core/src/main/java/org/apache/pinot/core/indexsegment/mutable/MutableSegmentImpl.java
+++
b/pinot-core/src/main/java/org/apache/pinot/core/indexsegment/mutable/MutableSegmentImpl.java
@@ -639,7 +639,8 @@ public class MutableSegmentImpl implements MutableSegment {
if (fieldSpec == null) {
// If the column was added during ingestion, we will construct the
column provider based on its fieldSpec to provide values
fieldSpec = _newlyAddedColumnsFieldMap.get(column);
-Preconditions.checkNotNull(fieldSpec, "FieldSpec for " + column + "
should not be null");
+Preconditions.checkNotNull(fieldSpec,
+"FieldSpec for " + column + " should not be null. " + "Potentially
invalid column name specified.");
}
// TODO: Refactor virtual column provider to directly generate data
source
VirtualColumnContext virtualColumnContext = new
VirtualColumnContext(fieldSpec, _numDocsIndexed);
-
To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org
For additional commands, e-mail: commits-h...@pinot.apache.org
[incubator-pinot] branch master updated (8900ebb -> a122728)
This is an automated email from the ASF dual-hosted git repository. kishoreg pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git. from 8900ebb Improve test coverage for TransformFunctionType (#5979) add a122728 [TE] Added support for BigQuery as data source (#5868) No new revisions were added by this update. Summary of changes: thirdeye/docs/bigquery.rst | 95 ++ thirdeye/docs/datasources.rst | 1 + thirdeye/install.sh| 34 +++- thirdeye/pom.xml | 50 +++- thirdeye/thirdeye-pinot/pom.xml| 28 +++ .../resources/v2/RootCauseTemplateResource.java| 5 +- .../datasource/sql/SqlResponseCacheLoader.java | 33 .../pinot/thirdeye/datasource/sql/SqlUtils.java| 23 ++ 8 files changed, 264 insertions(+), 5 deletions(-) create mode 100644 thirdeye/docs/bigquery.rst - To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
[incubator-pinot] branch zookeeper-put-api updated (418b6ad -> d3e86ad)
This is an automated email from the ASF dual-hosted git repository. kishoreg pushed a change to branch zookeeper-put-api in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git. discard 418b6ad Adding api to edit ZK path discard d6b9bef adding autocomplete in sql editor add d2ed761 adding autocomplete in sql editor (#5810) add ced24ba Adding api to edit ZK path add d3e86ad Adding delete api This update added new revisions after undoing existing revisions. That is to say, some revisions that were in the old version of the branch are not in the new version. This situation occurs when a user --force pushes a change and generates a repository containing something like this: * -- * -- B -- O -- O -- O (418b6ad) \ N -- N -- N refs/heads/zookeeper-put-api (d3e86ad) You should already have received notification emails for all of the O revisions, and so the following emails describe only the N revisions from the common base, B. Any revisions marked "omit" are not gone; other references still refer to them. Any revisions marked "discard" are gone forever. No new revisions were added by this update. Summary of changes: .../api/resources/ZookeeperResource.java | 24 ++ .../helix/core/PinotHelixResourceManager.java | 13 2 files changed, 33 insertions(+), 4 deletions(-) - To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
[incubator-pinot] branch master updated (a892fb4 -> d2ed761)
This is an automated email from the ASF dual-hosted git repository. kishoreg pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git. from a892fb4 Add Range Indexing support for raw values (#5853) add d2ed761 adding autocomplete in sql editor (#5810) No new revisions were added by this update. Summary of changes: .../src/main/resources/app/pages/Query.tsx | 161 +--- .../src/main/resources/app/styles/styles.css | 61 .../main/resources/app/utils/PinotMethodUtils.ts | 11 +- .../src/main/resources/app/utils/Utils.tsx | 163 - pinot-controller/src/main/resources/package.json | 1 + 5 files changed, 342 insertions(+), 55 deletions(-) - To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
[incubator-pinot] 01/02: adding autocomplete in sql editor
This is an automated email from the ASF dual-hosted git repository.
kishoreg pushed a commit to branch zookeeper-put-api
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
commit d6b9bef3a1ae10c0c7ec3707652e950c6cdca147
Author: Sanket Shah
AuthorDate: Wed Aug 5 19:54:37 2020 +0530
adding autocomplete in sql editor
---
.../src/main/resources/app/pages/Query.tsx | 161 +---
.../src/main/resources/app/styles/styles.css | 61
.../main/resources/app/utils/PinotMethodUtils.ts | 11 +-
.../src/main/resources/app/utils/Utils.tsx | 163 -
pinot-controller/src/main/resources/package.json | 1 +
5 files changed, 342 insertions(+), 55 deletions(-)
diff --git a/pinot-controller/src/main/resources/app/pages/Query.tsx
b/pinot-controller/src/main/resources/app/pages/Query.tsx
index fdb9b88..e3b67f5 100644
--- a/pinot-controller/src/main/resources/app/pages/Query.tsx
+++ b/pinot-controller/src/main/resources/app/pages/Query.tsx
@@ -29,6 +29,10 @@ import 'codemirror/lib/codemirror.css';
import 'codemirror/theme/material.css';
import 'codemirror/mode/javascript/javascript';
import 'codemirror/mode/sql/sql';
+import 'codemirror/addon/hint/show-hint';
+import 'codemirror/addon/hint/sql-hint';
+import 'codemirror/addon/hint/show-hint.css';
+import NativeCodeMirror from 'codemirror';
import _ from 'lodash';
import FormControlLabel from '@material-ui/core/FormControlLabel';
import Switch from '@material-ui/core/Switch';
@@ -40,6 +44,7 @@ import QuerySideBar from '../components/Query/QuerySideBar';
import TableToolbar from '../components/TableToolbar';
import SimpleAccordion from '../components/SimpleAccordion';
import PinotMethodUtils from '../utils/PinotMethodUtils';
+import '../styles/styles.css';
const useStyles = makeStyles((theme) => ({
title: {
@@ -48,7 +53,11 @@ const useStyles = makeStyles((theme) => ({
},
rightPanel: {},
codeMirror: {
-'& .CodeMirror': { height: 100, border: '1px solid #BDCCD9', fontSize:
'13px' },
+'& .CodeMirror': {
+ height: 100,
+ border: '1px solid #BDCCD9',
+ fontSize: '13px',
+},
},
queryOutput: {
'& .CodeMirror': { height: 430, border: '1px solid #BDCCD9' },
@@ -74,8 +83,8 @@ const useStyles = makeStyles((theme) => ({
marginBottom: '20px',
},
sqlError: {
-whiteSpace: 'pre-wrap'
- }
+whiteSpace: 'pre-wrap',
+ },
}));
const jsonoptions = {
@@ -85,16 +94,19 @@ const jsonoptions = {
gutters: ['CodeMirror-lint-markers'],
lint: true,
theme: 'default',
- readOnly: true
+ readOnly: true,
};
const sqloptions = {
lineNumbers: true,
- mode: 'sql',
+ mode: 'text/x-sql',
styleActiveLine: true,
- gutters: ['CodeMirror-lint-markers'],
lint: true,
- theme: 'default'
+ theme: 'default',
+ indentWithTabs: true,
+ smartIndent: true,
+ lineWrapping: true,
+ extraKeys: { "'@'": 'autocomplete' },
};
const QueryPage = () => {
@@ -125,13 +137,13 @@ const QueryPage = () => {
const [queryStats, setQueryStats] = useState({
columns: [],
-records: []
+records: [],
});
const [checked, setChecked] = React.useState({
tracing: false,
querySyntaxPQL: false,
-showResultJSON: false
+showResultJSON: false,
});
const [copyMsg, showCopyMsg] = React.useState(false);
@@ -162,10 +174,14 @@ const QueryPage = () => {
});
}
-const results = await PinotMethodUtils.getQueryResults(params, url,
checked);
+const results = await PinotMethodUtils.getQueryResults(
+ params,
+ url,
+ checked
+);
setResultError(results.error || '');
-setResultData(results.result || {columns: [], records: []});
-setQueryStats(results.queryStats || {columns: [], records: []});
+setResultData(results.result || { columns: [], records: [] });
+setQueryStats(results.queryStats || { columns: [], records: [] });
setOutputResult(JSON.stringify(results.data, null, 2) || '');
setQueryLoader(false);
};
@@ -223,6 +239,40 @@ const QueryPage = () => {
fetchData();
}, []);
+ const handleSqlHints = (cm: NativeCodeMirror.Editor) => {
+const tableNames = [];
+tableList.records.forEach((obj, i) => {
+ tableNames.push(obj[i]);
+});
+const columnNames = tableSchema.records.map((obj) => {
+ return obj[0];
+});
+const hintOptions = [];
+const defaultHint = (NativeCodeMirror as any).hint.sql(cm);
+
+Array.prototype.push.apply(hintOptions,
Utils.generateCodeM
[incubator-pinot] branch zookeeper-put-api created (now 418b6ad)
This is an automated email from the ASF dual-hosted git repository. kishoreg pushed a change to branch zookeeper-put-api in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git. at 418b6ad Adding api to edit ZK path This branch includes the following new commits: new d6b9bef adding autocomplete in sql editor new 418b6ad Adding api to edit ZK path The 2 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. - To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
[incubator-pinot] 02/02: Adding api to edit ZK path
This is an automated email from the ASF dual-hosted git repository.
kishoreg pushed a commit to branch zookeeper-put-api
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
commit 418b6ad24e53f5a177da5aec586e78ff99e94e33
Author: kishoreg
AuthorDate: Sun Aug 30 01:01:44 2020 -0700
Adding api to edit ZK path
---
.../api/resources/ZookeeperResource.java | 38 ++
.../helix/core/PinotHelixResourceManager.java | 4 +++
2 files changed, 42 insertions(+)
diff --git
a/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/ZookeeperResource.java
b/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/ZookeeperResource.java
index 3093052..f4c00ba 100644
---
a/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/ZookeeperResource.java
+++
b/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/ZookeeperResource.java
@@ -19,6 +19,7 @@
package org.apache.pinot.controller.api.resources;
import com.fasterxml.jackson.core.JsonProcessingException;
+import com.google.common.base.Charsets;
import io.swagger.annotations.Api;
import io.swagger.annotations.ApiOperation;
import io.swagger.annotations.ApiParam;
@@ -30,11 +31,14 @@ import java.util.Map;
import javax.inject.Inject;
import javax.ws.rs.DefaultValue;
import javax.ws.rs.GET;
+import javax.ws.rs.POST;
+import javax.ws.rs.PUT;
import javax.ws.rs.Path;
import javax.ws.rs.Produces;
import javax.ws.rs.QueryParam;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
+import org.apache.helix.AccessOption;
import org.apache.helix.ZNRecord;
import org.apache.helix.manager.zk.ZNRecordSerializer;
import org.apache.pinot.controller.helix.core.PinotHelixResourceManager;
@@ -75,6 +79,40 @@ public class ZookeeperResource {
return null;
}
+ @PUT
+ @Path("/zk/put")
+ @Produces(MediaType.TEXT_PLAIN)
+ @ApiOperation(value = "Get content of the znode")
+ @ApiResponses(value = { //
+ @ApiResponse(code = 200, message = "Success"), //
+ @ApiResponse(code = 404, message = "ZK Path not found"), //
+ @ApiResponse(code = 204, message = "No Content"), //
+ @ApiResponse(code = 500, message = "Internal server error")})
+ public SuccessResponse putData(
+ @ApiParam(value = "Zookeeper Path, must start with /", required = true,
defaultValue = "/") @QueryParam("path") @DefaultValue("") String path,
+ @ApiParam(value = "Content", required = true) @QueryParam("data")
@DefaultValue("") String content,
+ @ApiParam(value = "expectedVersion", required = true, defaultValue =
"-1") @QueryParam("expectedVersion") @DefaultValue("-1") String expectedVersion,
+ @ApiParam(value = "accessOption", required = true, defaultValue = "1")
@QueryParam("accessOption") @DefaultValue("1") String accessOption) {
+path = validateAndNormalizeZKPath(path);
+ZNRecord record = null;
+if (content != null) {
+ record = (ZNRecord)
_znRecordSerializer.deserialize(content.getBytes(Charsets.UTF_8));
+}
+try {
+ boolean result = pinotHelixResourceManager
+ .setZKData(path, record, Integer.parseInt(expectedVersion),
Integer.parseInt(accessOption));
+ if (result) {
+return new SuccessResponse("Successfully Updated path: " + path);
+ } else {
+throw new ControllerApplicationException(LOGGER, "Failed to update
path: " + path,
+Response.Status.INTERNAL_SERVER_ERROR);
+ }
+} catch (Exception e) {
+ throw new ControllerApplicationException(LOGGER, "Failed to update path:
" + path,
+ Response.Status.INTERNAL_SERVER_ERROR, e);
+}
+ }
+
@GET
@Path("/zk/ls")
@Produces(MediaType.APPLICATION_JSON)
diff --git
a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/PinotHelixResourceManager.java
b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/PinotHelixResourceManager.java
index 7b6c468..f07b6be 100644
---
a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/PinotHelixResourceManager.java
+++
b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/PinotHelixResourceManager.java
@@ -1191,6 +1191,10 @@ public class PinotHelixResourceManager {
}
}
+ public boolean setZKData(String path, ZNRecord record, int expectedVersion,
int accessOption) {
+return _helixDataAccessor.getBaseDataAccessor().set(path, record,
expectedVersion, accessOption);
+ }
+
public ZNRecord readZKData(String path) {
return _helixDataAccessor.getBaseDataAccessor().get(path, null, -1);
}
-
To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org
For additional commands, e-mail: commits-h...@pinot.apache.org
[incubator-pinot] branch exact-distinct-count updated (e5fd3a0 -> 8f2dff4)
This is an automated email from the ASF dual-hosted git repository. kishoreg pushed a change to branch exact-distinct-count in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git. from e5fd3a0 Removing code from another PR add 8f2dff4 addressing review comments No new revisions were added by this update. Summary of changes: .../apache/pinot/core/common/ObjectSerDeUtils.java | 26 +++--- 1 file changed, 23 insertions(+), 3 deletions(-) - To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
[incubator-pinot] branch exact-distinct-count updated (3c531f3 -> e5fd3a0)
This is an automated email from the ASF dual-hosted git repository. kishoreg pushed a change to branch exact-distinct-count in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git. from 3c531f3 Fix tests add e5fd3a0 Removing code from another PR No new revisions were added by this update. Summary of changes: .../java/org/apache/pinot/common/function/AggregationFunctionType.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) - To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
[incubator-pinot] branch master updated: [TE] frontend - harleyjj/rca - fix heatmap click bug (#5880)
This is an automated email from the ASF dual-hosted git repository.
kishoreg pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
The following commit(s) were added to refs/heads/master by this push:
new da1fe59 [TE] frontend - harleyjj/rca - fix heatmap click bug (#5880)
da1fe59 is described below
commit da1fe59027128453855d72a21ef0ba56772ef2ac
Author: Harley Jackson
AuthorDate: Mon Aug 17 10:21:16 2020 -0700
[TE] frontend - harleyjj/rca - fix heatmap click bug (#5880)
---
.../app/pods/components/heatmap-chart/component.js| 8
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git
a/thirdeye/thirdeye-frontend/app/pods/components/heatmap-chart/component.js
b/thirdeye/thirdeye-frontend/app/pods/components/heatmap-chart/component.js
index 2f31d91..b7b59d2 100644
--- a/thirdeye/thirdeye-frontend/app/pods/components/heatmap-chart/component.js
+++ b/thirdeye/thirdeye-frontend/app/pods/components/heatmap-chart/component.js
@@ -46,7 +46,7 @@ export default Component.extend({
role,
dimName,
dimValue
-} = subdimension;
+} = (subdimension.data || {});
if (!onInclude) { return; }
@@ -66,8 +66,8 @@ export default Component.extend({
role,
dimName,
dimValue
-} = subdimension;
-
+} = (subdimension.data || {});
+
if (!onExclude) { return; }
onExclude(role, dimName, dimValue);
@@ -130,7 +130,7 @@ export default Component.extend({
const nodes = treeMap({name: '0', children: children})
// specify children of treemap
.children
-// only nodes which don't have children
+// only nodes which don't have children
.filter((node) => !node.children);
this._createCell(div, nodes, tooltipId);
});
-
To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org
For additional commands, e-mail: commits-h...@pinot.apache.org
[incubator-pinot] branch exact-distinct-count updated (dc1b19e -> ef30e07)
This is an automated email from the ASF dual-hosted git repository.
kishoreg pushed a change to branch exact-distinct-count
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git.
discard dc1b19e Deleting file unrelated to this PR
discard a7ab0fd Support for exact distinct count for non int data types
add 6085578 Enable segment decryption for encrypted segments in Minion
(#5821)
add fd0130b Enhance sql parser for having and post-aggregation (#5825)
add 8a3cecb [TE] frontend - harleyjj/forecast - show tool tip for fit and
forecast (#5812)
add ce32362 Addressed issues in code review: (#5774)
add cc7a344 [TE] UI changes for anomalies page v3(#5824)
add 7f8e65c [TE] Fix template rendering errors (#5839)
add debadaa [TE] enchance anomaly api to propagate feedback (#5823)
add 63a4fd4 Clarifications in realtime provisioning helper (#5838)
add de14078 Support post-aggregation in QueryContext (#5827)
add 9f23e18 Tiered storage (#5793)
add 57d4b71 [TE] Presto JDBC lib upgrade (#5841)
add 3785aa7 Improving retention manager to handle segment lineage
clean-up (#5828)
add c04b8b3 Move quickstart tests to use Java 14 release (#5844)
add d8264c1 Fix data ingestion from Amazon S3 bucket (#5836)
add 9551062 update Swagger (OpenAPI) configuration for HTTP+HTTPS (#5817)
add b268012 [TE] add anomaly detection as a service - Phase 1 (#5769)
add 1c754f8 Added set-diff operators and changed distinctCountThetaSketch
syntax (#5832)
add 47323de [TE] Dashboard Resource Refactor (#5808)
add db48107 Adding controller healthcheck endpoint: /health (#5846)
add f4949e9 [TE] Added reset application API +refactor (#5847)
add bb8b19e DataGenerator to tolerate DATE_TIME and COMPLEX fields (#5848)
add d28c5cf Add pinot-spark-connector (#5787)
add 8ab032f Makes Pinot work on Alpine Linux or Distroless + BusyBox
(#5818)
add eb0f713 Fix encrypted file path in Segment Fetcher (#5854)
add 0b6ef98 Support multi-value non-dictionary group by (#5851)
add 2cfaed3 Support type conversion for all scalar functions (#5849)
add 09e9804 [TE] Fix wrong task pickup logic (#5855)
add 5469a84 add timeColumnName to tableConfig to enable TE auto-detection
(#5860)
add 449bf94 Fix NPE for aggregate metrics (#5862)
add 45d5d29 Add additional datetime functionality (#5438)
add 6dd54f8 Fix the variable names for off-heap alloc configs (#5852)
add 2b58bfb [TE] clean up legacy code (#5842)
add ee6e541 Support for exact distinct count for non int data types
add 2808c58 Deleting file unrelated to this PR
add ef30e07 Fixing serde for bytesset
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (dc1b19e)
\
N -- N -- N refs/heads/exact-distinct-count (ef30e07)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
No new revisions were added by this update.
Summary of changes:
.github/workflows/pinot_tests.yml | 2 +-
LICENSE-binary | 2 +-
.../broker/broker/BrokerAdminApiApplication.java | 4 +-
.../common/assignment/InstancePartitionsUtils.java | 23 +-
.../pinot/common/function/DateTimeFunctions.java | 262 ---
.../common/function/DateTimePatternHandler.java| 4 +-
.../apache/pinot/common/function/FunctionInfo.java | 44 +-
.../pinot/common/function/FunctionInvoker.java | 137 ++--
.../pinot/common/function/FunctionRegistry.java| 29 +-
.../pinot/common/function/FunctionUtils.java | 118
.../common/function/TransformFunctionType.java | 2 +-
.../function/annotations/ScalarFunction.java | 23 +-
.../function/scalar/ArithmeticFunctions.java | 95 +++
.../common/function/scalar/DateTimeFunctions.java | 555 +++
.../function/{ => scalar}/JsonFunctions.java | 9 +-
.../function/{ => scalar}/StringFunctions.java | 40 +-
.../pinot/common/metrics/ControllerMeter.java | 2 +
.../pinot/common/tier/PinotServerTierStorage.java | 31 +-
.../java/org/apache/pinot/common/tier/Tier.java| 50 ++
.../org/apache/pinot/common/tier/TierFactory.java | 59 ++
.../pinot/common/tier/TierSegmentSelector.java | 26 +-
.../org/apache/pinot/common/tier/TierStorage.java | 13 +-
.../common/tier/TimeBasedTierSegmentSelector.java | 83 +++
...
[incubator-pinot] branch exact-distinct-count updated (a7ab0fd -> dc1b19e)
This is an automated email from the ASF dual-hosted git repository. kishoreg pushed a change to branch exact-distinct-count in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git. from a7ab0fd Support for exact distinct count for non int data types add dc1b19e Deleting file unrelated to this PR No new revisions were added by this update. Summary of changes: .../DistinctRawBloomFilterAggregationFunction.java | 226 - 1 file changed, 226 deletions(-) delete mode 100644 pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/DistinctRawBloomFilterAggregationFunction.java - To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
[incubator-pinot] 01/01: Support for exact distinct count for non int data types
This is an automated email from the ASF dual-hosted git repository.
kishoreg pushed a commit to branch exact-distinct-count
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
commit a7ab0fd56978ca4412f86d6c09d86e1d3cb35baf
Author: kishoreg
AuthorDate: Sun Aug 16 02:28:55 2020 -0700
Support for exact distinct count for non int data types
---
.../common/function/AggregationFunctionType.java | 3 +-
.../apache/pinot/core/common/ObjectSerDeUtils.java | 176 +--
.../query/DictionaryBasedAggregationOperator.java | 24 +-
.../function/DistinctCountAggregationFunction.java | 243 -
.../DistinctCountMVAggregationFunction.java| 31 +--
.../DistinctRawBloomFilterAggregationFunction.java | 226 +++
6 files changed, 609 insertions(+), 94 deletions(-)
diff --git
a/pinot-common/src/main/java/org/apache/pinot/common/function/AggregationFunctionType.java
b/pinot-common/src/main/java/org/apache/pinot/common/function/AggregationFunctionType.java
index fc60ea6..b0db043 100644
---
a/pinot-common/src/main/java/org/apache/pinot/common/function/AggregationFunctionType.java
+++
b/pinot-common/src/main/java/org/apache/pinot/common/function/AggregationFunctionType.java
@@ -61,7 +61,8 @@ public enum AggregationFunctionType {
PERCENTILEMV("percentileMV"),
PERCENTILEESTMV("percentileEstMV"),
PERCENTILETDIGESTMV("percentileTDigestMV"),
- DISTINCT("distinct");
+ DISTINCT("distinct"),
+ DISTINCTRAWBLOOMFILTER("distinctRawBloomFilter");
private final String _name;
diff --git
a/pinot-core/src/main/java/org/apache/pinot/core/common/ObjectSerDeUtils.java
b/pinot-core/src/main/java/org/apache/pinot/core/common/ObjectSerDeUtils.java
index 9c87921..8995952 100644
---
a/pinot-core/src/main/java/org/apache/pinot/core/common/ObjectSerDeUtils.java
+++
b/pinot-core/src/main/java/org/apache/pinot/core/common/ObjectSerDeUtils.java
@@ -19,17 +19,31 @@
package org.apache.pinot.core.common;
import com.clearspring.analytics.stream.cardinality.HyperLogLog;
+import com.google.common.base.Charsets;
import com.google.common.primitives.Longs;
import com.tdunning.math.stats.MergingDigest;
import com.tdunning.math.stats.TDigest;
import it.unimi.dsi.fastutil.doubles.DoubleArrayList;
+import it.unimi.dsi.fastutil.doubles.DoubleIterator;
+import it.unimi.dsi.fastutil.doubles.DoubleOpenHashSet;
+import it.unimi.dsi.fastutil.doubles.DoubleSet;
+import it.unimi.dsi.fastutil.floats.FloatIterator;
+import it.unimi.dsi.fastutil.floats.FloatOpenHashSet;
+import it.unimi.dsi.fastutil.floats.FloatSet;
import it.unimi.dsi.fastutil.ints.IntIterator;
import it.unimi.dsi.fastutil.ints.IntOpenHashSet;
import it.unimi.dsi.fastutil.ints.IntSet;
+import it.unimi.dsi.fastutil.longs.LongIterator;
+import it.unimi.dsi.fastutil.longs.LongOpenHashSet;
+import it.unimi.dsi.fastutil.longs.LongSet;
+import it.unimi.dsi.fastutil.objects.ObjectIterator;
+import it.unimi.dsi.fastutil.objects.ObjectOpenHashSet;
+import it.unimi.dsi.fastutil.objects.ObjectSet;
import java.io.ByteArrayOutputStream;
import java.io.DataOutputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
+import java.nio.charset.Charset;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
@@ -68,7 +82,11 @@ public class ObjectSerDeUtils {
DistinctTable(11),
DataSketch(12),
Geometry(13),
-RoaringBitmap(14);
+RoaringBitmap(14),
+LongSet(15),
+FloatSet(16),
+DoubleSet(17),
+BytesSet(18);
private final int _value;
@@ -111,6 +129,14 @@ public class ObjectSerDeUtils {
return ObjectType.Geometry;
} else if (value instanceof RoaringBitmap) {
return ObjectType.RoaringBitmap;
+ } else if (value instanceof LongSet) {
+return ObjectType.LongSet;
+ } else if (value instanceof it.unimi.dsi.fastutil.floats.FloatSet) {
+return ObjectType.FloatSet;
+ } else if (value instanceof it.unimi.dsi.fastutil.doubles.DoubleSet) {
+return ObjectType.DoubleSet;
+ } else if (value instanceof ObjectSet) {
+return ObjectType.BytesSet;
} else {
throw new IllegalArgumentException("Unsupported type of value: " +
value.getClass().getSimpleName());
}
@@ -452,6 +478,135 @@ public class ObjectSerDeUtils {
}
};
+ public static final ObjectSerDe LONG_SET_SER_DE = new
ObjectSerDe() {
+
+@Override
+public byte[] serialize(LongSet longSet) {
+ int size = longSet.size();
+ byte[] bytes = new byte[Integer.BYTES + size * Long.BYTES];
+ ByteBuffer byteBuffer = ByteBuffer.wrap(bytes);
+ byteBuffer.putInt(size);
+ LongIterator iterator = longSet.iterator();
+ while (iterator.hasNext()) {
+byteBuffer.putLong(iterator.nextLong());
+ }
+ return bytes;
+}
+
+@Override
+public LongSet deserialize(byte[] bytes) {
+
[incubator-pinot] branch exact-distinct-count created (now a7ab0fd)
This is an automated email from the ASF dual-hosted git repository. kishoreg pushed a change to branch exact-distinct-count in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git. at a7ab0fd Support for exact distinct count for non int data types This branch includes the following new commits: new a7ab0fd Support for exact distinct count for non int data types The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. - To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
[incubator-pinot] branch master updated (1c754f8 -> 47323de)
This is an automated email from the ASF dual-hosted git repository. kishoreg pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git. from 1c754f8 Added set-diff operators and changed distinctCountThetaSketch syntax (#5832) add 47323de [TE] Dashboard Resource Refactor (#5808) No new revisions were added by this update. Summary of changes: .../api/application/ApplicationResource.java | 3 +- .../api/detection/AnomalyDetectionResource.java| 1 - .../api/user/dashboard/UserDashboardResource.java | 3 +- .../dashboard/ThirdEyeDashboardApplication.java| 60 +- .../dashboard/resources/AdminResource.java | 33 +++- .../resources/AnomalyFlattenResource.java | 3 +- .../dashboard/resources/AutoOnboardResource.java | 3 +- .../dashboard/resources/CacheResource.java | 3 +- .../resources/CustomizedEventResource.java | 3 +- .../dashboard/resources/DashboardResource.java | 71 +++ .../dashboard/resources/DatasetConfigResource.java | 3 +- .../dashboard/resources/EntityManagerResource.java | 3 +- .../dashboard/resources/EntityMappingResource.java | 3 +- .../dashboard/resources/MetricConfigResource.java | 3 +- .../resources/OnboardDatasetMetricResource.java| 3 +- .../thirdeye/dashboard/resources/RootResource.java | 216 + .../dashboard/resources/SummaryResource.java | 9 +- .../dashboard/resources/ThirdEyeResource.java | 25 ++- .../dashboard/resources/v2/AnomaliesResource.java | 3 +- .../dashboard/resources/v2/ConfigResource.java | 3 +- .../dashboard/resources/v2/DataResource.java | 3 +- .../resources/v2/DetectionAlertResource.java | 3 +- .../resources/v2/alerts/AlertResource.java | 4 +- .../v2/anomalies/AnomalySearchResource.java| 4 +- .../dataset/DatasetAutoOnboardResource.java| 3 +- .../sql/resources/SqlDataSourceResource.java | 3 +- .../detection/DetectionConfigurationResource.java | 3 +- .../thirdeye/detection/DetectionResource.java | 16 +- .../thirdeye/detection/yaml/YamlResource.java | 3 +- 29 files changed, 364 insertions(+), 132 deletions(-) create mode 100644 thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/dashboard/resources/RootResource.java - To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
[incubator-pinot] branch master updated (d8264c1 -> 9551062)
This is an automated email from the ASF dual-hosted git repository. kishoreg pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git. from d8264c1 Fix data ingestion from Amazon S3 bucket (#5836) add 9551062 update Swagger (OpenAPI) configuration for HTTP+HTTPS (#5817) No new revisions were added by this update. Summary of changes: LICENSE-binary | 2 +- .../pinot/broker/broker/BrokerAdminApiApplication.java | 4 ++-- .../org/apache/pinot/controller/ControllerConf.java | 9 - .../org/apache/pinot/controller/ControllerStarter.java | 2 +- .../controller/api/ControllerAdminApiApplication.java | 14 +- .../pinot/controller/util/ListenerConfigUtil.java | 17 - .../pinot/controller/util/ListenerConfigUtilTest.java | 6 -- .../pinot/server/starter/helix/AdminApiApplication.java | 4 ++-- .../service/PinotServiceManagerAdminApiApplication.java | 4 ++-- pom.xml | 2 +- 10 files changed, 14 insertions(+), 50 deletions(-) - To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
[incubator-pinot] branch master updated: Pradeep/sr ssl fix (#5758)
This is an automated email from the ASF dual-hosted git repository.
kishoreg pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 2e08602 Pradeep/sr ssl fix (#5758)
2e08602 is described below
commit 2e08602c086f02294714beaec761c2c6a9de7c30
Author: pradeepgv42 <66842697+pradeepg...@users.noreply.github.com>
AuthorDate: Thu Aug 6 10:23:27 2020 -0700
Pradeep/sr ssl fix (#5758)
* Stashing avro bug fixe
* Only add known ssl configs
* Add documentation for the new configs
Co-authored-by: Pradeep Gopanapalli Venkata
---
docs/pluggable_streams.rst | 56 ++
...aConfluentSchemaRegistryAvroMessageDecoder.java | 49 ++-
2 files changed, 104 insertions(+), 1 deletion(-)
diff --git a/docs/pluggable_streams.rst b/docs/pluggable_streams.rst
index 8cc231f..309c840 100644
--- a/docs/pluggable_streams.rst
+++ b/docs/pluggable_streams.rst
@@ -268,6 +268,62 @@ confluent schema registry:
}
}
+Here is another example which uses SSL based authentication to talk with kafka
+and schema-registry. Notice there are two sets of SSL options, ones starting
with
+`ssl.` are for kafka consumer and ones with
`stream.kafka.decoder.prop.schema.registry.`
+are for `SchemaRegistryClient` used by
`KafkaConfluentSchemaRegistryAvroMessageDecoder`.
+
+
+.. code-block:: none
+
+ {
+"tableName": "meetupRsvp",
+"tableType": "REALTIME",
+"segmentsConfig": {
+ "timeColumnName": "mtime",
+ "timeType": "MILLISECONDS",
+ "segmentPushType": "APPEND",
+ "segmentAssignmentStrategy": "BalanceNumSegmentAssignmentStrategy",
+ "schemaName": "meetupRsvp",
+ "replication": "1",
+ "replicasPerPartition": "1"
+},
+"tenants": {},
+"tableIndexConfig": {
+ "loadMode": "MMAP",
+ "streamConfigs": {
+"streamType": "kafka",
+"stream.kafka.consumer.type": "LowLevel",
+"stream.kafka.topic.name": "meetupRSVPEvents",
+"stream.kafka.decoder.class.name":
"org.apache.pinot.plugin.inputformat.avro.confluent.KafkaConfluentSchemaRegistryAvroMessageDecoder",
+"stream.kafka.consumer.factory.class.name":
"org.apache.pinot.plugin.stream.kafka20.KafkaConsumerFactory",
+"stream.kafka.zk.broker.url": "localhost:2191/kafka",
+"stream.kafka.broker.list": "localhost:19092",
+
+"schema.registry.url": "",
+"security.protocol": "",
+"ssl.truststore.location": "",
+"ssl.keystore.location": "",
+"ssl.truststore.password": "",
+"ssl.keystore.password": "",
+"ssl.key.password": "",
+
+"stream.kafka.decoder.prop.schema.registry.rest.url": "",
+"stream.kafka.decoder.prop.schema.registry.ssl.truststore.location":
"",
+"stream.kafka.decoder.prop.schema.registry.ssl.keystore.location": "",
+"stream.kafka.decoder.prop.schema.registry.ssl.truststore.password":
"",
+"stream.kafka.decoder.prop.schema.registry.ssl.keystore.password": "",
+"stream.kafka.decoder.prop.schema.registry.ssl.keystore.type": "",
+"stream.kafka.decoder.prop.schema.registry.ssl.truststore.type": "",
+"stream.kafka.decoder.prop.schema.registry.ssl.key.password": "",
+"stream.kafka.decoder.prop.schema.registry.ssl.protocol": "",
+ }
+},
+"metadata": {
+ "customConfigs": {}
+}
+ }
+
Upgrade from Kafka 0.9 connector to Kafka 2.x connector
---
diff --git
a/pinot-plugins/pinot-input-format/pinot-confluent-avro/src/main/java/org/apache/pinot/plugin/inputformat/avro/confluent/KafkaConfluentSchemaRegistryAvroMessageDecoder.java
b/pinot-plugins/pinot-input-format/pinot-confluent-avro/src/main/java/org/apache/pinot/plugin/inputformat/avro/confluent/KafkaConfluentSchemaRegistryAvroMessageDecoder.java
index 4e61d9a..1f0f442 100644
---
a/pinot-plugins/pinot-input-format/pinot-confluent-avro/src/main/java/org/apache/pinot/plugin/inputformat/avro/confluent/KafkaConfluentSchemaRegistryAvroMessageDecoder.java
+++
b/pinot-plugins/pinot-input-format/pin
[incubator-pinot] branch master updated: New endpoint to get routing table for sql query (#5791)
This is an automated email from the ASF dual-hosted git repository.
kishoreg pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
The following commit(s) were added to refs/heads/master by this push:
new d1b4586 New endpoint to get routing table for sql query (#5791)
d1b4586 is described below
commit d1b458644e505f07c919e7eb983feb9dafcd0061
Author: Oğuzhan Mangır
AuthorDate: Tue Aug 4 06:05:32 2020 +0300
New endpoint to get routing table for sql query (#5791)
* Add new endpoint to get routing table for sql query
* Add integration test for routing table sql endpoint
---
.../broker/api/resources/PinotBrokerDebug.java | 25 ++
.../tests/HybridClusterIntegrationTest.java| 15 -
2 files changed, 35 insertions(+), 5 deletions(-)
diff --git
a/pinot-broker/src/main/java/org/apache/pinot/broker/api/resources/PinotBrokerDebug.java
b/pinot-broker/src/main/java/org/apache/pinot/broker/api/resources/PinotBrokerDebug.java
index 1743dee..cb6f71a 100644
---
a/pinot-broker/src/main/java/org/apache/pinot/broker/api/resources/PinotBrokerDebug.java
+++
b/pinot-broker/src/main/java/org/apache/pinot/broker/api/resources/PinotBrokerDebug.java
@@ -42,13 +42,15 @@ import org.apache.pinot.core.transport.ServerInstance;
import org.apache.pinot.pql.parsers.Pql2Compiler;
import org.apache.pinot.spi.config.table.TableType;
import org.apache.pinot.spi.utils.builder.TableNameBuilder;
+import org.apache.pinot.sql.parsers.CalciteSqlCompiler;
@Api(tags = "Debug")
@Path("/")
// TODO: Add APIs to return the RoutingTable (with unavailable segments)
public class PinotBrokerDebug {
- private static final Pql2Compiler COMPILER = new Pql2Compiler();
+ private static final Pql2Compiler PQL_COMPILER = new Pql2Compiler();
+ private static final CalciteSqlCompiler CALCITE_COMPILER = new
CalciteSqlCompiler();
@Inject
private RoutingManager _routingManager;
@@ -82,7 +84,7 @@ public class PinotBrokerDebug {
if (tableType != TableType.REALTIME) {
String offlineTableName =
TableNameBuilder.OFFLINE.tableNameWithType(tableName);
RoutingTable routingTable =
-
_routingManager.getRoutingTable(COMPILER.compileToBrokerRequest("SELECT * FROM
" + offlineTableName));
+
_routingManager.getRoutingTable(PQL_COMPILER.compileToBrokerRequest("SELECT *
FROM " + offlineTableName));
if (routingTable != null) {
result.put(offlineTableName,
routingTable.getServerInstanceToSegmentsMap());
}
@@ -90,7 +92,7 @@ public class PinotBrokerDebug {
if (tableType != TableType.OFFLINE) {
String realtimeTableName =
TableNameBuilder.REALTIME.tableNameWithType(tableName);
RoutingTable routingTable =
-
_routingManager.getRoutingTable(COMPILER.compileToBrokerRequest("SELECT * FROM
" + realtimeTableName));
+
_routingManager.getRoutingTable(PQL_COMPILER.compileToBrokerRequest("SELECT *
FROM " + realtimeTableName));
if (routingTable != null) {
result.put(realtimeTableName,
routingTable.getServerInstanceToSegmentsMap());
}
@@ -109,11 +111,26 @@ public class PinotBrokerDebug {
@ApiResponses(value = {@ApiResponse(code = 200, message = "Routing table"),
@ApiResponse(code = 404, message = "Routing not found"), @ApiResponse(code =
500, message = "Internal server error")})
public Map> getRoutingTableForQuery(
@ApiParam(value = "Pql query (table name should have type suffix)")
@QueryParam("pql") String pql) {
-RoutingTable routingTable =
_routingManager.getRoutingTable(COMPILER.compileToBrokerRequest(pql));
+RoutingTable routingTable =
_routingManager.getRoutingTable(PQL_COMPILER.compileToBrokerRequest(pql));
if (routingTable != null) {
return routingTable.getServerInstanceToSegmentsMap();
} else {
throw new WebApplicationException("Cannot find routing for query: " +
pql, Response.Status.NOT_FOUND);
}
}
+
+ @GET
+ @Produces(MediaType.APPLICATION_JSON)
+ @Path("/debug/routingTable/sql")
+ @ApiOperation(value = "Get the routing table for a SQL query")
+ @ApiResponses(value = {@ApiResponse(code = 200, message = "Routing table"),
@ApiResponse(code = 404, message = "Routing not found"), @ApiResponse(code =
500, message = "Internal server error")})
+ public Map> getRoutingTableForSQLQuery(
+ @ApiParam(value = "SQL query (table name should have type suffix)")
@QueryParam("query") String query) {
+RoutingTable routingTable =
_routingManager.getRoutingTable(CALCITE_COMPILER.compileToBrokerRequest(query));
+if (routingTable != null) {
+ return routingTable.getServerInstanceToSegmentsMap();
+} else {
+ throw new W
[incubator-pinot] branch master updated (cb7de23 -> 0fc0811)
This is an automated email from the ASF dual-hosted git repository. kishoreg pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git. from cb7de23 [TE] Added a backfill start date for Anomaly Detection (#5740) add 0fc0811 add user info in url to auth header in HTTP getDownloadFileRequest (#5772) No new revisions were added by this update. Summary of changes: .../org/apache/pinot/common/utils/FileUploadDownloadClient.java| 7 +++ 1 file changed, 7 insertions(+) - To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
[incubator-pinot] branch master updated (bc2c066 -> a910c04)
This is an automated email from the ASF dual-hosted git repository. kishoreg pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git. from bc2c066 [TE] frontend - harleyjj/rca - Phase 1 of custom baseline selector (#5752) add a910c04 adding zookeeper browser module in UI (#5763) No new revisions were added by this update. Summary of changes: .../main/resources/app/components/AppLoader.tsx| 16 +- .../main/resources/app/components/Breadcrumbs.tsx | 3 +- .../src/main/resources/app/components/Layout.tsx | 4 +- .../main/resources/app/components/MaterialTree.tsx | 122 .../app/components/Query/QuerySideBar.tsx | 3 +- .../src/main/resources/app/components/SideBar.tsx | 2 +- .../app/components/SvgIcons/ZookeeperIcon.tsx | 44 + .../src/main/resources/app/components/TabPanel.tsx | 38 ++-- .../app/components/Zookeeper/TreeDirectory.tsx | 175 + pinot-controller/src/main/resources/app/index.html | 2 +- .../src/main/resources/app/interfaces/types.d.ts | 4 +- .../main/resources/app/pages/InstanceDetails.tsx | 2 +- .../src/main/resources/app/pages/Query.tsx | 6 +- .../src/main/resources/app/pages/ZookeeperPage.tsx | 207 + .../src/main/resources/app/requests/index.ts | 14 +- pinot-controller/src/main/resources/app/router.tsx | 4 +- .../src/main/resources/app/styles/styles.css | 2 +- .../src/main/resources/app/theme/typography.ts | 3 +- .../main/resources/app/utils/PinotMethodUtils.ts | 65 ++- .../src/main/resources/app/utils/Utils.tsx | 14 +- pinot-controller/src/main/resources/package.json | 1 + 21 files changed, 682 insertions(+), 49 deletions(-) create mode 100644 pinot-controller/src/main/resources/app/components/MaterialTree.tsx create mode 100644 pinot-controller/src/main/resources/app/components/SvgIcons/ZookeeperIcon.tsx copy website/src/components/CodeHeader/index.js => pinot-controller/src/main/resources/app/components/TabPanel.tsx (58%) create mode 100644 pinot-controller/src/main/resources/app/components/Zookeeper/TreeDirectory.tsx create mode 100644 pinot-controller/src/main/resources/app/pages/ZookeeperPage.tsx - To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
[incubator-pinot] branch distinct-count-bitmap created (now cbcc6a0)
This is an automated email from the ASF dual-hosted git repository. kishoreg pushed a change to branch distinct-count-bitmap in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git. at cbcc6a0 Adding distinct count support based on bitmap This branch includes the following new commits: new cbcc6a0 Adding distinct count support based on bitmap The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. - To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
[incubator-pinot] 01/01: Adding distinct count support based on bitmap
This is an automated email from the ASF dual-hosted git repository.
kishoreg pushed a commit to branch distinct-count-bitmap
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
commit cbcc6a0d0641ebc7ea99d7bda488599b56f5c845
Author: kishoreg
AuthorDate: Tue Jul 28 17:16:40 2020 -0700
Adding distinct count support based on bitmap
---
.../common/function/AggregationFunctionType.java | 1 +
.../apache/pinot/core/common/ObjectSerDeUtils.java | 62 +++-
.../DistinctCountBitmapValueAggregator.java| 95 ++
.../function/AggregationFunctionFactory.java | 2 +
.../function/AggregationFunctionVisitorBase.java | 3 +
.../function/DistinctCountAggregationFunction.java | 71 +
.../DistinctCountBitmapAggregationFunction.java| 354 +
7 files changed, 572 insertions(+), 16 deletions(-)
diff --git
a/pinot-common/src/main/java/org/apache/pinot/common/function/AggregationFunctionType.java
b/pinot-common/src/main/java/org/apache/pinot/common/function/AggregationFunctionType.java
index ff3fb50..6c7ebe5 100644
---
a/pinot-common/src/main/java/org/apache/pinot/common/function/AggregationFunctionType.java
+++
b/pinot-common/src/main/java/org/apache/pinot/common/function/AggregationFunctionType.java
@@ -28,6 +28,7 @@ public enum AggregationFunctionType {
MINMAXRANGE("minMaxRange"),
DISTINCTCOUNT("distinctCount"),
DISTINCTCOUNTHLL("distinctCountHLL"),
+ DISTINCTCOUNTBITMAP("distinctCountBitmap"),
DISTINCTCOUNTRAWHLL("distinctCountRawHLL"),
FASTHLL("fastHLL"),
DISTINCTCOUNTTHETASKETCH("distinctCountThetaSketch"),
diff --git
a/pinot-core/src/main/java/org/apache/pinot/core/common/ObjectSerDeUtils.java
b/pinot-core/src/main/java/org/apache/pinot/core/common/ObjectSerDeUtils.java
index f471e37..5cbe20f 100644
---
a/pinot-core/src/main/java/org/apache/pinot/core/common/ObjectSerDeUtils.java
+++
b/pinot-core/src/main/java/org/apache/pinot/core/common/ObjectSerDeUtils.java
@@ -40,6 +40,8 @@ import
org.apache.pinot.core.query.aggregation.function.customobject.AvgPair;
import
org.apache.pinot.core.query.aggregation.function.customobject.DistinctTable;
import
org.apache.pinot.core.query.aggregation.function.customobject.MinMaxRangePair;
import
org.apache.pinot.core.query.aggregation.function.customobject.QuantileDigest;
+import org.roaringbitmap.buffer.ImmutableRoaringBitmap;
+import org.roaringbitmap.buffer.MutableRoaringBitmap;
/**
@@ -63,7 +65,8 @@ public class ObjectSerDeUtils {
IntSet(9),
TDigest(10),
DistinctTable(11),
-DataSketch(12);
+DataSketch(12),
+Bitmap(13);
private int _value;
@@ -102,6 +105,8 @@ public class ObjectSerDeUtils {
return ObjectType.DistinctTable;
} else if (value instanceof Sketch) {
return ObjectType.DataSketch;
+ } else if (value instanceof MutableRoaringBitmap) {
+return ObjectType.Bitmap;
} else {
throw new IllegalArgumentException("Unsupported type of value: " +
value.getClass().getSimpleName());
}
@@ -286,6 +291,44 @@ public class ObjectSerDeUtils {
}
};
+ public static final ObjectSerDe ROARING_BITMAP_SERDE =
new ObjectSerDe() {
+
+@Override
+public byte[] serialize(MutableRoaringBitmap bitmap) {
+ try {
+byte[] bytes = new byte[bitmap.serializedSizeInBytes()];
+bitmap.serialize(ByteBuffer.wrap(bytes));
+return bytes;
+ } catch (Exception e) {
+throw new RuntimeException("Caught exception while serializing
RoaringBitmap", e);
+ }
+}
+
+@Override
+public MutableRoaringBitmap deserialize(byte[] bytes) {
+ try {
+MutableRoaringBitmap bitmap = new MutableRoaringBitmap();
+bitmap.deserialize(ByteBuffer.wrap(bytes));
+return bitmap;
+ } catch (IOException e) {
+throw new RuntimeException("Caught exception while de-serializing
MutableRoaringBitmap", e);
+ }
+}
+
+@Override
+public MutableRoaringBitmap deserialize(ByteBuffer byteBuffer) {
+ byte[] bytes = new byte[byteBuffer.remaining()];
+ byteBuffer.get(bytes);
+ try {
+MutableRoaringBitmap bitmap = new MutableRoaringBitmap();
+bitmap.deserialize(ByteBuffer.wrap(bytes));
+return bitmap;
+ } catch (IOException e) {
+throw new RuntimeException("Caught exception while de-serializing
MutableRoaringBitmap", e);
+ }
+}
+ };
+
public static final ObjectSerDe DISTINCT_TABLE_SER_DE = new
ObjectSerDe() {
@Override
@@ -484,21 +527,8 @@ public class ObjectSerDeUtils {
// NOTE: DO NOT change the order, it has to be the same order as the
ObjectType
//@formatter:off
- private static final ObjectSerDe[] SER_DES = {
- STRING_SER_DE,
- LONG_SER_DE,
- DOUBLE_SER_DE,
- DOUBLE_ARRAY_LIST_SER_DE,
-
[incubator-pinot] branch master updated: Pradeep/s3 credential chain fix (#5755)
This is an automated email from the ASF dual-hosted git repository.
kishoreg pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
The following commit(s) were added to refs/heads/master by this push:
new d437487 Pradeep/s3 credential chain fix (#5755)
d437487 is described below
commit d437487b08fd61ce5449bc5cfa9169352f54c6d5
Author: pradeepgv42 <66842697+pradeepg...@users.noreply.github.com>
AuthorDate: Sat Jul 25 12:47:00 2020 -0700
Pradeep/s3 credential chain fix (#5755)
* Add ProfileCredentialsProvider & instanceProfileCredentialsProvider
to the credentials provider chain for S3PinotFs
* Use a generic CredentialProvider and fix a bug in isDirectory func
* Remove HeadObjectRequest infavor of ListObjectsV2Request which limits the
number of keys returned back
Co-authored-by: Pradeep Gopanapalli Venkata
---
.../apache/pinot/plugin/filesystem/S3PinotFS.java | 22 +-
1 file changed, 5 insertions(+), 17 deletions(-)
diff --git
a/pinot-plugins/pinot-file-system/pinot-s3/src/main/java/org/apache/pinot/plugin/filesystem/S3PinotFS.java
b/pinot-plugins/pinot-file-system/pinot-s3/src/main/java/org/apache/pinot/plugin/filesystem/S3PinotFS.java
index 2e504ea..54f7912 100644
---
a/pinot-plugins/pinot-file-system/pinot-s3/src/main/java/org/apache/pinot/plugin/filesystem/S3PinotFS.java
+++
b/pinot-plugins/pinot-file-system/pinot-s3/src/main/java/org/apache/pinot/plugin/filesystem/S3PinotFS.java
@@ -41,10 +41,8 @@ import com.google.common.collect.ImmutableList;
import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
-import software.amazon.awssdk.auth.credentials.AwsCredentialsProviderChain;
-import
software.amazon.awssdk.auth.credentials.EnvironmentVariableCredentialsProvider;
+import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider;
import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
-import
software.amazon.awssdk.auth.credentials.SystemPropertyCredentialsProvider;
import software.amazon.awssdk.core.sync.RequestBody;
import software.amazon.awssdk.core.sync.ResponseTransformer;
import software.amazon.awssdk.regions.Region;
@@ -89,9 +87,7 @@ public class S3PinotFS extends PinotFS {
AwsBasicCredentials awsBasicCredentials =
AwsBasicCredentials.create(accessKey, secretKey);
awsCredentialsProvider =
StaticCredentialsProvider.create(awsBasicCredentials);
} else {
-awsCredentialsProvider =
-
AwsCredentialsProviderChain.builder().addCredentialsProvider(SystemPropertyCredentialsProvider.create())
-
.addCredentialsProvider(EnvironmentVariableCredentialsProvider.create()).build();
+awsCredentialsProvider = DefaultCredentialsProvider.create();
}
_s3Client =
S3Client.builder().region(Region.of(region)).credentialsProvider(awsCredentialsProvider).build();
@@ -434,18 +430,10 @@ public class S3PinotFS extends PinotFS {
if (prefix.equals(DELIMITER)) {
return true;
}
- try {
-HeadObjectRequest headObjectRequest =
-
HeadObjectRequest.builder().bucket(uri.getHost()).key(uri.getPath()).build();
-HeadObjectResponse s3ObjectMetadata =
_s3Client.headObject(headObjectRequest);
-
-return s3ObjectMetadata.sdkHttpResponse().isSuccessful();
- } catch (NoSuchKeyException e) {
-LOGGER.error("Could not get directory entry for {}", uri);
- }
- ListObjectsV2Request listObjectsV2Request =
-
ListObjectsV2Request.builder().bucket(uri.getHost()).prefix(prefix).build();
+ ListObjectsV2Request listObjectsV2Request = ListObjectsV2Request
+ .builder().bucket(uri.getHost())
+ .prefix(prefix).maxKeys(2).build();
ListObjectsV2Response listObjectsV2Response =
_s3Client.listObjectsV2(listObjectsV2Request);
return listObjectsV2Response.hasContents();
} catch (NoSuchKeyException e) {
-
To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org
For additional commands, e-mail: commits-h...@pinot.apache.org
[incubator-pinot] branch master updated (dc01cc8 -> 8474f7e)
This is an automated email from the ASF dual-hosted git repository. kishoreg pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git. from dc01cc8 Fixing the case of Orc and Json record reader class name (#5754) add 8474f7e Cluster Manger UI: adding instance details page showing instance config and tables in that particular instance (#5757) No new revisions were added by this update. Summary of changes: pinot-controller/src/main/resources/app/App.tsx| 58 --- .../main/resources/app/components/Breadcrumbs.tsx | 22 ++- .../app/components/Homepage/InstanceTable.tsx | 12 +- .../app/components/Homepage/InstancesTables.tsx| 8 +- .../src/main/resources/app/interfaces/types.d.ts | 2 + .../main/resources/app/pages/InstanceDetails.tsx | 167 + .../src/main/resources/app/pages/Query.tsx | 7 +- .../src/main/resources/app/pages/TenantDetails.tsx | 6 +- .../src/main/resources/app/requests/index.ts | 9 +- pinot-controller/src/main/resources/app/router.tsx | 8 +- .../main/resources/app/utils/PinotMethodUtils.ts | 67 +++-- 11 files changed, 311 insertions(+), 55 deletions(-) create mode 100644 pinot-controller/src/main/resources/app/pages/InstanceDetails.tsx - To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
[incubator-pinot] branch master updated: updated cluster manage UI and added table details page and segment details page (#5732)
This is an automated email from the ASF dual-hosted git repository.
kishoreg pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 2e16aa4 updated cluster manage UI and added table details page and
segment details page (#5732)
2e16aa4 is described below
commit 2e16aa4676198f44c7a9532d1d229125069e567b
Author: Sanket Shah
AuthorDate: Thu Jul 23 13:00:38 2020 +0530
updated cluster manage UI and added table details page and segment details
page (#5732)
* updated cluster manage UI and added table details page and segment
details page
* showing error message if sql query returns any exception
---
.../main/resources/app/components/Breadcrumbs.tsx | 11 +-
.../app/components/Homepage/ClusterConfig.tsx | 24 +-
.../app/components/Homepage/InstanceTable.tsx | 42 +-
.../app/components/Homepage/InstancesTables.tsx| 26 +-
.../app/components/Homepage/TenantsTable.tsx | 43 +-
.../src/main/resources/app/components/Layout.tsx | 7 +-
.../app/components/Query/QuerySideBar.tsx | 7 +-
.../main/resources/app/components/SearchBar.tsx| 14 +-
.../resources/app/components/SimpleAccordion.tsx | 96 +
.../app/components/SvgIcons/ClusterManagerIcon.tsx | 32 ++
.../app/components/SvgIcons/QueryConsoleIcon.tsx | 36 +-
.../src/main/resources/app/components/Table.tsx| 249 ++-
.../{EnhancedTableToolbar.tsx => TableToolbar.tsx} | 12 +-
.../src/main/resources/app/interfaces/types.d.ts | 23 +-
.../src/main/resources/app/pages/Query.tsx | 298 +++--
.../main/resources/app/pages/SegmentDetails.tsx| 170
.../src/main/resources/app/pages/TenantDetails.tsx | 139 ++-
.../src/main/resources/app/pages/Tenants.tsx | 64 +--
.../src/main/resources/app/requests/index.ts | 16 +-
pinot-controller/src/main/resources/app/router.tsx | 6 +-
.../main/resources/app/utils/PinotMethodUtils.ts | 462 +
.../src/main/resources/app/utils/Utils.tsx | 25 ++
pinot-controller/src/main/resources/package.json | 3 +-
23 files changed, 1375 insertions(+), 430 deletions(-)
diff --git a/pinot-controller/src/main/resources/app/components/Breadcrumbs.tsx
b/pinot-controller/src/main/resources/app/components/Breadcrumbs.tsx
index 0cdf09a..0df6f25 100644
--- a/pinot-controller/src/main/resources/app/components/Breadcrumbs.tsx
+++ b/pinot-controller/src/main/resources/app/components/Breadcrumbs.tsx
@@ -85,13 +85,14 @@ const BreadcrumbsComponent = ({ ...props }) => {
const breadcrumbs = [getClickableLabel(breadcrumbNameMap['/'], '/')];
const paramsKeys = _.keys(props.match.params);
if(paramsKeys.length){
-const {tenantName, tableName} = props.match.params;
-if(!tableName && tenantName){
- breadcrumbs.push(getLabel(tenantName));
-} else {
+const {tenantName, tableName, segmentName} = props.match.params;
+if(tenantName && tableName){
breadcrumbs.push(getClickableLabel(tenantName,
`/tenants/${tenantName}`));
- breadcrumbs.push(getLabel(tableName));
}
+if(tenantName && tableName && segmentName){
+ breadcrumbs.push(getClickableLabel(tableName,
`/tenants/${tenantName}/table/${tableName}`));
+}
+breadcrumbs.push(getLabel(segmentName || tableName || tenantName));
} else {
breadcrumbs.push(getLabel(breadcrumbNameMap[location.pathname]));
}
diff --git
a/pinot-controller/src/main/resources/app/components/Homepage/ClusterConfig.tsx
b/pinot-controller/src/main/resources/app/components/Homepage/ClusterConfig.tsx
index 8b65e59..1b5217c 100644
---
a/pinot-controller/src/main/resources/app/components/Homepage/ClusterConfig.tsx
+++
b/pinot-controller/src/main/resources/app/components/Homepage/ClusterConfig.tsx
@@ -19,9 +19,9 @@
import React, { useEffect, useState } from 'react';
import { TableData } from 'Models';
-import { getClusterConfig } from '../../requests';
import AppLoader from '../AppLoader';
import CustomizedTables from '../Table';
+import PinotMethodUtils from '../../utils/PinotMethodUtils';
const ClusterConfig = () => {
@@ -31,21 +31,23 @@ const ClusterConfig = () => {
records: []
});
+ const fetchData = async () => {
+const result = await PinotMethodUtils.getClusterConfigData();
+setTableData(result);
+setFetching(false);
+ };
useEffect(() => {
-getClusterConfig().then(({ data }) => {
- setTableData({
-columns: ['Property', 'Value'],
-records: [
- ...Object.keys(data).map(key => [key, data[key]])
-]
- });
- setFetching(false);
-});
+fetchData();
[incubator-pinot] branch master updated: UI Enhancement: Pinot UI expandable leftmost column(Page Links) (#5723)
This is an automated email from the ASF dual-hosted git repository.
kishoreg pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 31fdc5d UI Enhancement: Pinot UI expandable leftmost column(Page
Links) (#5723)
31fdc5d is described below
commit 31fdc5d2baeec53edc8b42e5fccdab458c68ab4a
Author: Sanket Shah
AuthorDate: Wed Jul 22 12:04:22 2020 +0530
UI Enhancement: Pinot UI expandable leftmost column(Page Links) (#5723)
---
.../src/main/resources/app/components/Header.tsx | 14 ++-
.../src/main/resources/app/components/Layout.tsx | 24 -
.../src/main/resources/app/components/Logo.tsx | 37
.../src/main/resources/app/components/SideBar.tsx | 101 -
.../resources/app/components/SvgIcons/Logo.tsx | 60
.../app/components/SvgIcons/QueryConsoleIcon.tsx | 48 ++
.../app/components/SvgIcons/SwaggerIcon.tsx| 40
7 files changed, 257 insertions(+), 67 deletions(-)
diff --git a/pinot-controller/src/main/resources/app/components/Header.tsx
b/pinot-controller/src/main/resources/app/components/Header.tsx
index 1dcf7db..08817b0 100644
--- a/pinot-controller/src/main/resources/app/components/Header.tsx
+++ b/pinot-controller/src/main/resources/app/components/Header.tsx
@@ -20,20 +20,26 @@
import React from 'react';
import { Link } from 'react-router-dom';
import { AppBar, Box } from '@material-ui/core';
-import Logo from './Logo';
+import MenuIcon from '@material-ui/icons/Menu';
+import Logo from './SvgIcons/Logo';
import BreadcrumbsComponent from './Breadcrumbs';
type Props = {
highlightSidebarLink: (id: number) => void;
+ showHideSideBarHandler: () => void;
+ openSidebar: boolean;
};
-const Header = ({ highlightSidebarLink, ...props }: Props) => (
+const Header = ({ highlightSidebarLink, showHideSideBarHandler, openSidebar,
...props }: Props) => (
-
- highlightSidebarLink(1)}
/>
+
+ highlightSidebarLink(1)}
fulllogo={openSidebar.toString()} />
+
+ showHideSideBarHandler()} />
+
diff --git a/pinot-controller/src/main/resources/app/components/Layout.tsx
b/pinot-controller/src/main/resources/app/components/Layout.tsx
index 373fc15..b958bdc 100644
--- a/pinot-controller/src/main/resources/app/components/Layout.tsx
+++ b/pinot-controller/src/main/resources/app/components/Layout.tsx
@@ -21,11 +21,13 @@ import * as React from 'react';
import { Grid } from '@material-ui/core';
import Sidebar from './SideBar';
import Header from './Header';
+import QueryConsoleIcon from './SvgIcons/QueryConsoleIcon';
+import SwaggerIcon from './SvgIcons/SwaggerIcon';
const navigationItems = [
// { id: 1, name: 'Cluster Manager', link: '/' },
- { id: 1, name: 'Query Console', link: '/', },
- { id: 2, name: 'Swagger REST API', link: 'help', target: '_blank' },
+ { id: 1, name: 'Query Console', link: '/', icon: },
+ { id: 2, name: 'Swagger REST API', link: 'help', target: '_blank', icon:
}
];
const Layout = (props) => {
@@ -33,19 +35,33 @@ const Layout = (props) => {
const routeObj = navigationItems.find((obj)=>{ return obj.link === hash;});
const [selectedId, setSelectedId] = React.useState(routeObj?.id || 1);
+ const sidebarOpenState = !(localStorage.getItem('pinot_ui:sidebarState') ===
'false');
+ const [openSidebar, setOpenSidebar] = React.useState(sidebarOpenState);
const highlightSidebarLink = (id: number) => {
setSelectedId(id);
};
+
+ const showHideSideBarHandler = () => {
+const newSidebarState = !openSidebar;
+localStorage.setItem('pinot_ui:sidebarState', newSidebarState.toString());
+setOpenSidebar(newSidebarState);
+ };
+
return (
-
+
diff --git a/pinot-controller/src/main/resources/app/components/Logo.tsx
b/pinot-controller/src/main/resources/app/components/Logo.tsx
deleted file mode 100644
index 63be44f..000
--- a/pinot-controller/src/main/resources/app/components/Logo.tsx
+++ /dev/null
@@ -1,37 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- *
[incubator-pinot] branch master updated (e5ddff4 -> e8a5708)
This is an automated email from the ASF dual-hosted git repository. kishoreg pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git. from e5ddff4 Support BYTES type for dictinctCount (#5701) add e8a5708 [TE] Added a getting started doc to setup ThirdEye with MySQL persistence (#5689) No new revisions were added by this update. Summary of changes: thirdeye/docs/Makefile| 8 +- thirdeye/docs/README.md | 8 +- thirdeye/docs/getting_started.rst | 162 ++ thirdeye/docs/introduction.rst| 3 +- thirdeye/docs/mysql.rst | 41 -- thirdeye/docs/requirements.txt| 3 + 6 files changed, 211 insertions(+), 14 deletions(-) create mode 100644 thirdeye/docs/getting_started.rst create mode 100644 thirdeye/docs/requirements.txt - To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
[incubator-pinot] branch master updated: Adding Controller API to explore Zookeeper (#5687)
This is an automated email from the ASF dual-hosted git repository.
kishoreg pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 4378e87 Adding Controller API to explore Zookeeper (#5687)
4378e87 is described below
commit 4378e87aba652a727076b535f77f4c0bbd4473d2
Author: Kishore Gopalakrishna
AuthorDate: Mon Jul 13 22:34:38 2020 -0700
Adding Controller API to explore Zookeeper (#5687)
* Adding Zookeeper resource to browse ZK data via controller API
* Fixing api documentation
* Fixing api documentation
* Fixing api documentation
* Addressing comments
---
.../pinot/controller/api/resources/Constants.java | 1 +
.../api/resources/PinotClusterConfigs.java | 17 ++-
.../api/resources/ZookeeperResource.java | 165 +
.../helix/core/PinotHelixResourceManager.java | 36 -
4 files changed, 214 insertions(+), 5 deletions(-)
diff --git
a/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/Constants.java
b/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/Constants.java
index 13d5f88..5e463e0 100644
---
a/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/Constants.java
+++
b/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/Constants.java
@@ -39,6 +39,7 @@ public class Constants {
public static final String TASK_TAG = "Task";
public static final String LEAD_CONTROLLER_TAG = "Leader";
public static final String TABLE_NAME = "tableName";
+ public static final String ZOOKEEPER = "Zookeeper";
public static TableType validateTableType(String tableTypeStr) {
if (tableTypeStr == null || tableTypeStr.isEmpty()) {
diff --git
a/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotClusterConfigs.java
b/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotClusterConfigs.java
index a9c20ed..2825f55 100644
---
a/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotClusterConfigs.java
+++
b/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotClusterConfigs.java
@@ -58,6 +58,17 @@ public class PinotClusterConfigs {
PinotHelixResourceManager pinotHelixResourceManager;
@GET
+ @Path("/cluster/info")
+ @Produces(MediaType.APPLICATION_JSON)
+ @ApiOperation(value = "Get cluster Info", notes = "Get cluster Info")
+ @ApiResponses(value = {@ApiResponse(code = 200, message = "Success"),
@ApiResponse(code = 500, message = "Internal server error")})
+ public String getClusterInfo() {
+ObjectNode ret = JsonUtils.newObjectNode();
+ret.put("clusterName", pinotHelixResourceManager.getHelixClusterName());
+return ret.toString();
+ }
+
+ @GET
@Path("/cluster/configs")
@Produces(MediaType.APPLICATION_JSON)
@ApiOperation(value = "List cluster configurations", notes = "List cluster
level configurations")
@@ -94,9 +105,11 @@ public class PinotClusterConfigs {
}
return new SuccessResponse("Updated cluster config.");
} catch (IOException e) {
- throw new ControllerApplicationException(LOGGER, "Error converting
request to cluster config.", Response.Status.BAD_REQUEST, e);
+ throw new ControllerApplicationException(LOGGER, "Error converting
request to cluster config.",
+ Response.Status.BAD_REQUEST, e);
} catch (Exception e) {
- throw new ControllerApplicationException(LOGGER, "Failed to update
cluster config.", Response.Status.INTERNAL_SERVER_ERROR, e);
+ throw new ControllerApplicationException(LOGGER, "Failed to update
cluster config.",
+ Response.Status.INTERNAL_SERVER_ERROR, e);
}
}
diff --git
a/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/ZookeeperResource.java
b/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/ZookeeperResource.java
new file mode 100644
index 000..3093052
--- /dev/null
+++
b/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/ZookeeperResource.java
@@ -0,0 +1,165 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in wri
[incubator-pinot] branch zk-browser updated (d29a7f6 -> 7ce78f7)
This is an automated email from the ASF dual-hosted git repository. kishoreg pushed a change to branch zk-browser in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git. from d29a7f6 Fixing api documentation add 7ce78f7 Addressing comments No new revisions were added by this update. Summary of changes: .../pinot/controller/api/resources/PinotClusterConfigs.java | 2 +- .../apache/pinot/controller/api/resources/ZookeeperResource.java | 8 2 files changed, 5 insertions(+), 5 deletions(-) - To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
[incubator-pinot] branch master updated (020bb74 -> 3b88eff)
This is an automated email from the ASF dual-hosted git repository.
kishoreg pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git.
from 020bb74 adding column data type to result set (#5679)
add 3b88eff Adding support for JDBC (#5602)
No new revisions were added by this update.
Summary of changes:
.../pom.xml| 18 +-
.../org/apache/pinot/client/PinotConnection.java | 101 ++
.../pinot/client/PinotConnectionMetaData.java | 194
.../java/org/apache/pinot/client/PinotDriver.java | 99 ++
.../java/org/apache/pinot/client/PinotMeta.java| 72 ++
.../pinot/client/PinotPreparedStatement.java | 234 +
.../apache/pinot/client/PinotResultMetadata.java | 80 ++
.../org/apache/pinot/client/PinotResultSet.java| 434 +
.../org/apache/pinot/client/PinotStatement.java| 124 +++
.../pinot/client/base/AbstractBaseConnection.java | 342 +++
.../base/AbstractBaseConnectionMetaData.java | 1018 +++
.../client/base/AbstractBasePreparedStatement.java | 311 ++
.../pinot/client/base/AbstractBaseResultSet.java | 1023
.../client/base/AbstractBaseResultSetMetadata.java | 138 +++
.../pinot/client/base/AbstractBaseStatement.java | 241 +
.../controller/PinotControllerTransport.java | 95 ++
.../client/controller/request/SchemaRequest.java | 18 +-
.../client/controller/request/TableRequest.java|5 +-
.../response/ControllerResponseFuture.java | 84 ++
.../client/controller/response/SchemaResponse.java | 91 ++
.../client/controller/response/TableResponse.java | 97 ++
.../org/apache/pinot/client/utils/Constants.java | 59 ++
.../apache/pinot/client/utils/DateTimeUtils.java | 79 ++
.../org/apache/pinot/client/utils/DriverUtils.java | 148 +++
.../src/main/resources/java.sql.Driver |1 +
.../pinot/client/DummyPinotClientTransport.java| 65 ++
.../client/DummyPinotClientTransportFactory.java} | 16 +-
.../apache/pinot/client/PinotConnectionTest.java | 51 +
.../org/apache/pinot/client/PinotDriverTest.java | 61 ++
.../pinot/client/PinotPreparedStatementTest.java | 122 +++
.../apache/pinot/client/PinotResultSetTest.java| 222 +
.../apache/pinot/client/PinotStatementTest.java| 54 ++
.../src/test/resources}/selection.json |0
pinot-clients/pom.xml |1 +
pom.xml|3 +
35 files changed, 5680 insertions(+), 21 deletions(-)
copy pinot-clients/{pinot-java-client => pinot-jdbc-client}/pom.xml (86%)
create mode 100644
pinot-clients/pinot-jdbc-client/src/main/java/org/apache/pinot/client/PinotConnection.java
create mode 100644
pinot-clients/pinot-jdbc-client/src/main/java/org/apache/pinot/client/PinotConnectionMetaData.java
create mode 100644
pinot-clients/pinot-jdbc-client/src/main/java/org/apache/pinot/client/PinotDriver.java
create mode 100644
pinot-clients/pinot-jdbc-client/src/main/java/org/apache/pinot/client/PinotMeta.java
create mode 100644
pinot-clients/pinot-jdbc-client/src/main/java/org/apache/pinot/client/PinotPreparedStatement.java
create mode 100644
pinot-clients/pinot-jdbc-client/src/main/java/org/apache/pinot/client/PinotResultMetadata.java
create mode 100644
pinot-clients/pinot-jdbc-client/src/main/java/org/apache/pinot/client/PinotResultSet.java
create mode 100644
pinot-clients/pinot-jdbc-client/src/main/java/org/apache/pinot/client/PinotStatement.java
create mode 100644
pinot-clients/pinot-jdbc-client/src/main/java/org/apache/pinot/client/base/AbstractBaseConnection.java
create mode 100644
pinot-clients/pinot-jdbc-client/src/main/java/org/apache/pinot/client/base/AbstractBaseConnectionMetaData.java
create mode 100644
pinot-clients/pinot-jdbc-client/src/main/java/org/apache/pinot/client/base/AbstractBasePreparedStatement.java
create mode 100644
pinot-clients/pinot-jdbc-client/src/main/java/org/apache/pinot/client/base/AbstractBaseResultSet.java
create mode 100644
pinot-clients/pinot-jdbc-client/src/main/java/org/apache/pinot/client/base/AbstractBaseResultSetMetadata.java
create mode 100644
pinot-clients/pinot-jdbc-client/src/main/java/org/apache/pinot/client/base/AbstractBaseStatement.java
create mode 100644
pinot-clients/pinot-jdbc-client/src/main/java/org/apache/pinot/client/controller/PinotControllerTransport.java
copy
pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/SuccessResponse.java
=>
pinot-clients/pinot-jdbc-client/src/main/java/org/apache/pinot/client/controller/request/SchemaRequest.java
(74%)
copy
pinot-spi/src/main/java/org/apache/pinot/spi/data/readers/RecordReaderConfig.java
=>
pinot-clients/pinot-jdbc-client/src/main/java/org/apache/pinot/client/controller/request/TableRequest.java
(90%)
create mode 100644
pinot-clients/p
[incubator-pinot] branch master updated: adding column data type to result set (#5679)
This is an automated email from the ASF dual-hosted git repository.
kishoreg pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 020bb74 adding column data type to result set (#5679)
020bb74 is described below
commit 020bb743569ac241d686da43de5e39681fe3571b
Author: Kartik Khare
AuthorDate: Sun Jul 12 12:10:05 2020 +0530
adding column data type to result set (#5679)
Co-authored-by: Kartik Khare
---
.../src/main/java/org/apache/pinot/client/AbstractResultSet.java | 6 ++
.../src/main/java/org/apache/pinot/client/ResultSet.java | 8
.../main/java/org/apache/pinot/client/ResultTableResultSet.java | 5 +
3 files changed, 19 insertions(+)
diff --git
a/pinot-clients/pinot-java-client/src/main/java/org/apache/pinot/client/AbstractResultSet.java
b/pinot-clients/pinot-java-client/src/main/java/org/apache/pinot/client/AbstractResultSet.java
index 87cb98c..e1dbdd2 100644
---
a/pinot-clients/pinot-java-client/src/main/java/org/apache/pinot/client/AbstractResultSet.java
+++
b/pinot-clients/pinot-java-client/src/main/java/org/apache/pinot/client/AbstractResultSet.java
@@ -22,6 +22,12 @@ package org.apache.pinot.client;
* Shared implementation between the different ResultSets.
*/
abstract class AbstractResultSet implements ResultSet {
+
+ @Override
+ public String getColumnDataType(int columnIndex) {
+return null;
+ }
+
@Override
public int getInt(int rowIndex) {
return getInt(rowIndex, 0);
diff --git
a/pinot-clients/pinot-java-client/src/main/java/org/apache/pinot/client/ResultSet.java
b/pinot-clients/pinot-java-client/src/main/java/org/apache/pinot/client/ResultSet.java
index 05e1e21..502d8e4 100644
---
a/pinot-clients/pinot-java-client/src/main/java/org/apache/pinot/client/ResultSet.java
+++
b/pinot-clients/pinot-java-client/src/main/java/org/apache/pinot/client/ResultSet.java
@@ -45,6 +45,14 @@ public interface ResultSet {
String getColumnName(int columnIndex);
/**
+ * Returns the column type at a given index.
+ *
+ * @param columnIndex The index of the column for which to retrieve the name
+ * @return The data type of the column at the given column index. null if
data type is not supported
+ */
+ String getColumnDataType(int columnIndex);
+
+ /**
* Obtains the integer value for the given row.
*
* @param rowIndex The index of the row
diff --git
a/pinot-clients/pinot-java-client/src/main/java/org/apache/pinot/client/ResultTableResultSet.java
b/pinot-clients/pinot-java-client/src/main/java/org/apache/pinot/client/ResultTableResultSet.java
index 86a7c17..a913622 100644
---
a/pinot-clients/pinot-java-client/src/main/java/org/apache/pinot/client/ResultTableResultSet.java
+++
b/pinot-clients/pinot-java-client/src/main/java/org/apache/pinot/client/ResultTableResultSet.java
@@ -54,6 +54,11 @@ class ResultTableResultSet extends AbstractResultSet {
}
@Override
+ public String getColumnDataType(int columnIndex) {
+return _columnDataTypesArray.get(columnIndex).asText();
+ }
+
+ @Override
public String getString(int rowIndex, int columnIndex) {
JsonNode jsonValue = _rowsArray.get(rowIndex).get(columnIndex);
if (jsonValue.isTextual()) {
-
To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org
For additional commands, e-mail: commits-h...@pinot.apache.org
[incubator-pinot] branch zk-browser updated: Fixing api documentation
This is an automated email from the ASF dual-hosted git repository.
kishoreg pushed a commit to branch zk-browser
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
The following commit(s) were added to refs/heads/zk-browser by this push:
new d29a7f6 Fixing api documentation
d29a7f6 is described below
commit d29a7f6d0b449c1e15899607063577c27e14f961
Author: kishoreg
AuthorDate: Sat Jul 11 23:34:18 2020 -0700
Fixing api documentation
---
.../pinot/controller/api/resources/ZookeeperResource.java | 10 +-
1 file changed, 5 insertions(+), 5 deletions(-)
diff --git
a/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/ZookeeperResource.java
b/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/ZookeeperResource.java
index 6870adc..1daff80 100644
---
a/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/ZookeeperResource.java
+++
b/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/ZookeeperResource.java
@@ -57,14 +57,14 @@ public class ZookeeperResource {
@GET
@Path("/zookeeper/get")
@Produces(MediaType.TEXT_PLAIN)
- @ApiOperation(value = "Get content on the znode")
+ @ApiOperation(value = "Get content of the znode")
@ApiResponses(value = { //
@ApiResponse(code = 200, message = "Success"), //
@ApiResponse(code = 404, message = "ZK Path not found"), //
@ApiResponse(code = 204, message = "No Content"), //
@ApiResponse(code = 500, message = "Internal server error")})
public String getData(
- @ApiParam(value = "Zookeeper Path, must start with /", required = false,
defaultValue = "/") @QueryParam("path") @DefaultValue("") String path) {
+ @ApiParam(value = "Zookeeper Path, must start with /", required = true,
defaultValue = "/") @QueryParam("path") @DefaultValue("") String path) {
path = validateAndNormalizeZKPath(path);
@@ -84,7 +84,7 @@ public class ZookeeperResource {
@ApiResponse(code = 404, message = "ZK Path not found"), //
@ApiResponse(code = 500, message = "Internal server error")})
public String ls(
- @ApiParam(value = "Zookeeper Path, must start with /", required = false,
defaultValue = "/") @QueryParam("path") @DefaultValue("") String path) {
+ @ApiParam(value = "Zookeeper Path, must start with /", required = true,
defaultValue = "/") @QueryParam("path") @DefaultValue("") String path) {
path = validateAndNormalizeZKPath(path);
@@ -105,7 +105,7 @@ public class ZookeeperResource {
@ApiResponse(code = 404, message = "ZK Path not found"), //
@ApiResponse(code = 500, message = "Internal server error")})
public String lsl(
- @ApiParam(value = "Zookeeper Path, must start with /", required = false,
defaultValue = "/") @QueryParam("path") @DefaultValue("") String path) {
+ @ApiParam(value = "Zookeeper Path, must start with /", required = true,
defaultValue = "/") @QueryParam("path") @DefaultValue("") String path) {
path = validateAndNormalizeZKPath(path);
@@ -127,7 +127,7 @@ public class ZookeeperResource {
@ApiResponse(code = 404, message = "Table not found"), //
@ApiResponse(code = 500, message = "Internal server error")})
public String stat(
- @ApiParam(value = "Zookeeper Path, must start with /", required = false,
defaultValue = "/") @QueryParam("path") @DefaultValue("") String path) {
+ @ApiParam(value = "Zookeeper Path, must start with /", required = true,
defaultValue = "/") @QueryParam("path") @DefaultValue("") String path) {
path = validateAndNormalizeZKPath(path);
-
To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org
For additional commands, e-mail: commits-h...@pinot.apache.org
[incubator-pinot] branch zk-browser updated: Fixing api documentation
This is an automated email from the ASF dual-hosted git repository.
kishoreg pushed a commit to branch zk-browser
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
The following commit(s) were added to refs/heads/zk-browser by this push:
new ae16d8b Fixing api documentation
ae16d8b is described below
commit ae16d8b7231a12362844657bacdec4705dfaf45c
Author: kishoreg
AuthorDate: Sat Jul 11 23:27:17 2020 -0700
Fixing api documentation
---
.../main/java/org/apache/pinot/controller/api/resources/Constants.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git
a/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/Constants.java
b/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/Constants.java
index 05de8b8..5e463e0 100644
---
a/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/Constants.java
+++
b/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/Constants.java
@@ -39,7 +39,7 @@ public class Constants {
public static final String TASK_TAG = "Task";
public static final String LEAD_CONTROLLER_TAG = "Leader";
public static final String TABLE_NAME = "tableName";
- public static final String ZOOKEEPER = "zookeeper";
+ public static final String ZOOKEEPER = "Zookeeper";
public static TableType validateTableType(String tableTypeStr) {
if (tableTypeStr == null || tableTypeStr.isEmpty()) {
-
To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org
For additional commands, e-mail: commits-h...@pinot.apache.org
[incubator-pinot] branch zk-browser updated: Fixing api documentation
This is an automated email from the ASF dual-hosted git repository.
kishoreg pushed a commit to branch zk-browser
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
The following commit(s) were added to refs/heads/zk-browser by this push:
new 95fabab Fixing api documentation
95fabab is described below
commit 95fabab8ad93ef91a4f74022dd070b8c73f4fe13
Author: kishoreg
AuthorDate: Sat Jul 11 23:26:01 2020 -0700
Fixing api documentation
---
.../apache/pinot/controller/api/resources/ZookeeperResource.java | 8
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git
a/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/ZookeeperResource.java
b/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/ZookeeperResource.java
index 4ed1ba1..6870adc 100644
---
a/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/ZookeeperResource.java
+++
b/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/ZookeeperResource.java
@@ -57,7 +57,7 @@ public class ZookeeperResource {
@GET
@Path("/zookeeper/get")
@Produces(MediaType.TEXT_PLAIN)
- @ApiOperation(value = "Get the data of the specific path", notes = "Get the
data of the specific path")
+ @ApiOperation(value = "Get content on the znode")
@ApiResponses(value = { //
@ApiResponse(code = 200, message = "Success"), //
@ApiResponse(code = 404, message = "ZK Path not found"), //
@@ -78,7 +78,7 @@ public class ZookeeperResource {
@GET
@Path("/zookeeper/ls")
@Produces(MediaType.APPLICATION_JSON)
- @ApiOperation(value = "Listing the child nodes of one path", notes =
"Listing the child nodes of one path")
+ @ApiOperation(value = "List the child znodes")
@ApiResponses(value = { //
@ApiResponse(code = 200, message = "Success"), //
@ApiResponse(code = 404, message = "ZK Path not found"), //
@@ -99,7 +99,7 @@ public class ZookeeperResource {
@GET
@Path("/zookeeper/lsl")
@Produces(MediaType.APPLICATION_JSON)
- @ApiOperation(value = "Listing the child nodes of one path along with
stats", notes = "Listing the child nodes of one path along with stats")
+ @ApiOperation(value = "List the child znodes along with Stats")
@ApiResponses(value = { //
@ApiResponse(code = 200, message = "Success"), //
@ApiResponse(code = 404, message = "ZK Path not found"), //
@@ -121,7 +121,7 @@ public class ZookeeperResource {
@GET
@Path("/zookeeper/stat")
@Produces(MediaType.TEXT_PLAIN)
- @ApiOperation(value = "", notes = "Listing the child nodes of one path")
+ @ApiOperation(value = "Get the stat", notes = " Use this api to fetch
additional details of a znode such as creation time, modified time, numChildren
etc ")
@ApiResponses(value = { //
@ApiResponse(code = 200, message = "Success"), //
@ApiResponse(code = 404, message = "Table not found"), //
-
To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org
For additional commands, e-mail: commits-h...@pinot.apache.org
[incubator-pinot] branch zk-browser created (now b00a654)
This is an automated email from the ASF dual-hosted git repository. kishoreg pushed a change to branch zk-browser in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git. at b00a654 Adding Zookeeper resource to browse ZK data via controller API This branch includes the following new commits: new b00a654 Adding Zookeeper resource to browse ZK data via controller API The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. - To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
[incubator-pinot] 01/01: Adding Zookeeper resource to browse ZK data via controller API
This is an automated email from the ASF dual-hosted git repository.
kishoreg pushed a commit to branch zk-browser
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
commit b00a65444d8e3d99e496d44e7950122208022b00
Author: kishoreg
AuthorDate: Sat Jul 11 23:14:13 2020 -0700
Adding Zookeeper resource to browse ZK data via controller API
---
.../pinot/controller/api/resources/Constants.java | 1 +
.../api/resources/PinotClusterConfigs.java | 17 ++-
.../api/resources/ZookeeperResource.java | 165 +
.../helix/core/PinotHelixResourceManager.java | 36 -
4 files changed, 214 insertions(+), 5 deletions(-)
diff --git
a/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/Constants.java
b/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/Constants.java
index 13d5f88..05de8b8 100644
---
a/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/Constants.java
+++
b/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/Constants.java
@@ -39,6 +39,7 @@ public class Constants {
public static final String TASK_TAG = "Task";
public static final String LEAD_CONTROLLER_TAG = "Leader";
public static final String TABLE_NAME = "tableName";
+ public static final String ZOOKEEPER = "zookeeper";
public static TableType validateTableType(String tableTypeStr) {
if (tableTypeStr == null || tableTypeStr.isEmpty()) {
diff --git
a/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotClusterConfigs.java
b/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotClusterConfigs.java
index a9c20ed..3fbccdd 100644
---
a/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotClusterConfigs.java
+++
b/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotClusterConfigs.java
@@ -58,6 +58,17 @@ public class PinotClusterConfigs {
PinotHelixResourceManager pinotHelixResourceManager;
@GET
+ @Path("/cluster/info")
+ @Produces(MediaType.APPLICATION_JSON)
+ @ApiOperation(value = "Get cluster Ingo", notes = "Get cluster Info")
+ @ApiResponses(value = {@ApiResponse(code = 200, message = "Success"),
@ApiResponse(code = 500, message = "Internal server error")})
+ public String getClusterInfo() {
+ObjectNode ret = JsonUtils.newObjectNode();
+ret.put("clusterName", pinotHelixResourceManager.getHelixClusterName());
+return ret.toString();
+ }
+
+ @GET
@Path("/cluster/configs")
@Produces(MediaType.APPLICATION_JSON)
@ApiOperation(value = "List cluster configurations", notes = "List cluster
level configurations")
@@ -94,9 +105,11 @@ public class PinotClusterConfigs {
}
return new SuccessResponse("Updated cluster config.");
} catch (IOException e) {
- throw new ControllerApplicationException(LOGGER, "Error converting
request to cluster config.", Response.Status.BAD_REQUEST, e);
+ throw new ControllerApplicationException(LOGGER, "Error converting
request to cluster config.",
+ Response.Status.BAD_REQUEST, e);
} catch (Exception e) {
- throw new ControllerApplicationException(LOGGER, "Failed to update
cluster config.", Response.Status.INTERNAL_SERVER_ERROR, e);
+ throw new ControllerApplicationException(LOGGER, "Failed to update
cluster config.",
+ Response.Status.INTERNAL_SERVER_ERROR, e);
}
}
diff --git
a/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/ZookeeperResource.java
b/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/ZookeeperResource.java
new file mode 100644
index 000..4ed1ba1
--- /dev/null
+++
b/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/ZookeeperResource.java
@@ -0,0 +1,165 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.controller.api.resources;
+
+import com.fasterxml.jackson.cor
[incubator-pinot] branch master updated: Add benchmark documentation. (#5683)
This is an automated email from the ASF dual-hosted git repository.
kishoreg pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 9477e01 Add benchmark documentation. (#5683)
9477e01 is described below
commit 9477e014ed7e053e9d098b15742dbbdfe7e85493
Author: Nikola Grcevski <620+grcev...@users.noreply.github.com>
AuthorDate: Sat Jul 11 15:05:26 2020 -0400
Add benchmark documentation. (#5683)
---
contrib/pinot-druid-benchmark/README.md | 297
1 file changed, 297 insertions(+)
diff --git a/contrib/pinot-druid-benchmark/README.md
b/contrib/pinot-druid-benchmark/README.md
new file mode 100644
index 000..f7ac5f1
--- /dev/null
+++ b/contrib/pinot-druid-benchmark/README.md
@@ -0,0 +1,297 @@
+# Running the benchmark
+
+For instructions on how to run the Pinot/Druid benchmark please refer to the
+```run_benchmark.sh``` file.
+
+In order to run the Apache Pinot benchmark you'll need to create the
appropriate
+data segments, which are too large to be included in this github repository and
+they may need to be recreated with new Apache Pinot versions.
+
+To create the neccessary segment data for the benchmark please follow the
+instructions below.
+
+# Creating Apache Pinot benchmark segments from TPC-H data
+
+To run the Pinot/Druid benchmark with Apache Pinot you'll need to download and
run
+the TPC-H tools to generate the benchmark data sets.
+
+## Downloading and building the TPC-H tools
+
+The TPC-H tools can be downloaded from the [TPC-H
Website](http://www.tpc.org/tpch/default5.asp).
+Registration is required.
+
+**Note:**: The instructions below for dbgen assume a Linux OS.
+
+After downloading and extracing the TPC-H tools, you'll need to build the
+db generator tool: ```dbgen```. To do so, extract the package that you have
+downloaded from TPC-H's website and inside the dbgen sub directory edit the
+```makefile``` file.
+
+Set the following variables in the makefile to:
+
+```
+CC = gcc
+...
+DATABASE= SQLSERVER
+MACHINE = LINUX
+WORKLOAD = TPCH
+```
+
+Next, build the dbgen tool as per the README instructions in the dbgen
directory.
+
+## Generating the TPC-H data and converting them for use in Apache Pinot
+
+After building ```dbgen``` run the following command line in the ```dbgen```
directory:
+
+```
+./dbgen -TL -s8
+```
+
+The command above will generate a single large file called ```lineitem.tbl```.
+This is the data file for the TPC-H benchmark, which we'll need to
post-process
+a bit to be imported into Apache Pinot.
+
+Next, build the Pinot/Druid Benchmark code if you haven't done so already.
+
+**Note:** Apache Pinot has JDK11 support, however for now it's
+best to use JDK8 for all build and run operations in this manual.
+
+Inside ```pinot_directory/contrib/pinot-druid-benchmark``` run:
+
+```
+mvn clean install
+```
+
+Next, inside the same directory split the ```lineitem``` table:
+
+```
+./target/appassembler/bin/data-separator.sh
+```
+
+Use the output directory from the split as the input directory for the merge
+command below:
+
+```
+./target/appassembler/bin/data-merger.sh
YEAR
+```
+
+If all ran well you should see a few CSV files produced, 1992.csv through
1998.csv.
+
+These files are the starting point for creating our Apache Pinot segments.
+
+## Create the Apache Pinot segments
+
+The first step in the process is to launch a standalone Apache Pinot Cluster
on one
+single server. This cluster will serve as a host to hold the initial segments,
+which we'll extract and copy for later re-use in the benchmark.
+
+Follow the steps outlined in the Apache Pinot Manual Cluster setup to launch
the
+cluster:
+
+https://docs.pinot.apache.org/basics/getting-started/advanced-pinot-setup
+
+You don't need the Kafka service as we won't be using it.
+
+Next, we need to follow the instructions similar to the ones described in
+the [Batch Import
Example](https://docs.pinot.apache.org/basics/getting-started/pushing-your-data-to-pinot)
+in the Apache Pinot documentation.
+
+### Create the Apache Pinot tables
+
+Run:
+
+```
+pinot-admin.sh AddTable \
+ -tableConfigFile /absolute/path/to/table-config.json \
+ -schemaFile /absolute/path/to/schema.json -exec
+```
+
+For this command above you'll need the following configuration files:
+
+```table_config.json```
+```
+{
+ "tableName": "tpch_lineitem",
+ "segmentsConfig" : {
+"replication" : "1",
+"schemaName" : "tpch_lineitem",
+"segmentAssignmentStrategy" : "BalanceNumSegmentAssignmentStrategy"
+ },
+ "tenants" : {
+"broker":"DefaultTenant",
+"server":"DefaultTenant"
+ },
+ "tableIndexConfig" : {
+"starTree
[incubator-pinot] branch master updated: [TE] Updated README in ThirdEye Documentation (#5678)
This is an automated email from the ASF dual-hosted git repository. kishoreg pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git The following commit(s) were added to refs/heads/master by this push: new 07938ea [TE] Updated README in ThirdEye Documentation (#5678) 07938ea is described below commit 07938eae1357e3908ce624677728d555ed95a126 Author: Suvodeep Pyne AuthorDate: Fri Jul 10 11:09:55 2020 -0700 [TE] Updated README in ThirdEye Documentation (#5678) Changes - Added links to install sphinx which is required to build the docs - Refactored the readme and added some useful links Co-authored-by: Suvodeep Pyne --- thirdeye/docs/README.md | 31 ++- 1 file changed, 26 insertions(+), 5 deletions(-) diff --git a/thirdeye/docs/README.md b/thirdeye/docs/README.md index 5f23fcc..3e7fcc4 100644 --- a/thirdeye/docs/README.md +++ b/thirdeye/docs/README.md @@ -19,10 +19,31 @@ --> -Updating the docs: +# ThirdEye Documentation + +This directory contains the documentation for ThirdEye. It is available online at https://thirdeye.readthedocs.io. + +### Building docs + +The documentation is built using the [sphinx](https://www.sphinx-doc.org/) framework. + +If you have python/pip, you can install sphinx using +``` +pip install sphinx sphinx_rtd_theme +``` + +Build docs using +``` +make html +``` +The rendered html can be found at `_build/html/index.html` + +### Updating docs 1. Edit or add files as needed. -2. Run "make html" -3. Open _build/html/index.html in your favorite browser and ensure contents and links work correctly. +2. Build using `make html` +3. Open `_build/html/index.html` in your favorite browser +4. Ensure the contents and links work correctly +5. Submit a PR! + -NOTE: -You may see some differences locally as the version of sphinx-build on your local host might not be the same as the one used in readthedocs.io for building the pinot website docs. \ No newline at end of file +> NOTE: You may see some differences locally as the version of `sphinx-build` on your local host might not be the same as the one used in [readthedocs.io](https://readthedocs.io) for building the pinot website docs. \ No newline at end of file - To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
[incubator-pinot] branch master updated (9196fcb -> 2dc72a5)
This is an automated email from the ASF dual-hosted git repository. kishoreg pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git. from 9196fcb fetch all columns and datatypes (#5673) add 2dc72a5 Fixing code to fetch the fsConfig from the right parent (#5669) No new revisions were added by this update. Summary of changes: .../hadoop/HadoopSegmentGenerationJobRunner.java | 2 +- .../apache/pinot/spi/env/PinotConfiguration.java | 8 +- .../pinot/spi/filesystem/PinotFSFactory.java | 21 ++-- .../pinot/spi/env/PinotConfigurationTest.java | 139 - .../pinot/spi/filesystem/PinotFSFactoryTest.java | 13 ++ .../resources/pinot-configuration-1.properties | 8 +- 6 files changed, 171 insertions(+), 20 deletions(-) - To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
[incubator-pinot] branch pinot-fs-bug-fix updated (c703924 -> e3900cf)
This is an automated email from the ASF dual-hosted git repository. kishoreg pushed a change to branch pinot-fs-bug-fix in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git. from c703924 addressing comments add e3900cf Pinot fs bug fix (#5674) No new revisions were added by this update. Summary of changes: .../hadoop/HadoopSegmentGenerationJobRunner.java | 2 +- .../apache/pinot/spi/env/PinotConfiguration.java | 8 +- .../pinot/spi/env/PinotConfigurationTest.java | 139 - .../resources/pinot-configuration-1.properties | 8 +- 4 files changed, 148 insertions(+), 9 deletions(-) - To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
[incubator-pinot] branch master updated (1a420b2 -> 3497cc3)
This is an automated email from the ASF dual-hosted git repository. kishoreg pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git. from 1a420b2 Add integration test for theta sketches (#5514) add 3497cc3 Pinot S3Fs fix (#5670) No new revisions were added by this update. Summary of changes: pinot-distribution/pinot-assembly.xml| 4 pinot-plugins/pinot-file-system/pinot-s3/pom.xml | 4 ++-- pom.xml | 4 ++-- 3 files changed, 8 insertions(+), 4 deletions(-) - To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
[incubator-pinot] branch pinot-fs-bug-fix updated (d9745c9 -> c703924)
This is an automated email from the ASF dual-hosted git repository. kishoreg pushed a change to branch pinot-fs-bug-fix in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git. from d9745c9 Adding test case add c703924 addressing comments No new revisions were added by this update. Summary of changes: .../org/apache/pinot/spi/filesystem/PinotFSFactory.java | 17 - 1 file changed, 8 insertions(+), 9 deletions(-) - To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
[incubator-pinot] branch pinot-fs-bug-fix updated: Adding test case
This is an automated email from the ASF dual-hosted git repository.
kishoreg pushed a commit to branch pinot-fs-bug-fix
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
The following commit(s) were added to refs/heads/pinot-fs-bug-fix by this push:
new d9745c9 Adding test case
d9745c9 is described below
commit d9745c9a0f3f809bdeff7830be2ad92e28782367
Author: kishoreg
AuthorDate: Wed Jul 8 17:43:11 2020 -0700
Adding test case
---
.../org/apache/pinot/spi/filesystem/PinotFSFactoryTest.java | 13 +
1 file changed, 13 insertions(+)
diff --git
a/pinot-spi/src/test/java/org/apache/pinot/spi/filesystem/PinotFSFactoryTest.java
b/pinot-spi/src/test/java/org/apache/pinot/spi/filesystem/PinotFSFactoryTest.java
index b8853d7..9490e04 100644
---
a/pinot-spi/src/test/java/org/apache/pinot/spi/filesystem/PinotFSFactoryTest.java
+++
b/pinot-spi/src/test/java/org/apache/pinot/spi/filesystem/PinotFSFactoryTest.java
@@ -42,18 +42,26 @@ public class PinotFSFactoryTest {
public void testCustomizedSegmentFetcherFactory() {
Map properties = new HashMap<>();
properties.put("class.file", LocalPinotFS.class.getName());
+
properties.put("class.test", TestPinotFS.class.getName());
+properties.put("test.accessKey", "v1");
+properties.put("test.secretKey", "V2");
+properties.put("test.region", "us-east");
PinotFSFactory.init(new PinotConfiguration(properties));
PinotFS testPinotFS = PinotFSFactory.create("test");
Assert.assertTrue(testPinotFS instanceof TestPinotFS);
Assert.assertEquals(((TestPinotFS) testPinotFS).getInitCalled(), 1);
+Assert.assertEquals(((TestPinotFS)
testPinotFS).getConfiguration().getProperty("accessKey"), "v1");
+Assert.assertEquals(((TestPinotFS)
testPinotFS).getConfiguration().getProperty("secretKey"), "V2");
+Assert.assertEquals(((TestPinotFS)
testPinotFS).getConfiguration().getProperty("region"), "us-east");
Assert.assertTrue(PinotFSFactory.create("file") instanceof LocalPinotFS);
}
public static class TestPinotFS extends PinotFS {
public int initCalled = 0;
+private PinotConfiguration _configuration;
public int getInitCalled() {
return initCalled;
@@ -61,9 +69,14 @@ public class PinotFSFactoryTest {
@Override
public void init(PinotConfiguration configuration) {
+ _configuration = configuration;
initCalled++;
}
+public PinotConfiguration getConfiguration() {
+ return _configuration;
+}
+
@Override
public boolean mkdir(URI uri) {
return true;
-
To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org
For additional commands, e-mail: commits-h...@pinot.apache.org
[incubator-pinot] branch pinot-fs-bug-fix updated: Fixing code to fetch the fsConfig from the right parent
This is an automated email from the ASF dual-hosted git repository.
kishoreg pushed a commit to branch pinot-fs-bug-fix
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
The following commit(s) were added to refs/heads/pinot-fs-bug-fix by this push:
new 4c3eed8 Fixing code to fetch the fsConfig from the right parent
4c3eed8 is described below
commit 4c3eed8655a3b5d1cd6f702af2f7541b5c688ce1
Author: kishoreg
AuthorDate: Wed Jul 8 17:34:23 2020 -0700
Fixing code to fetch the fsConfig from the right parent
---
.../java/org/apache/pinot/spi/filesystem/PinotFSFactory.java | 12 ++--
1 file changed, 6 insertions(+), 6 deletions(-)
diff --git
a/pinot-spi/src/main/java/org/apache/pinot/spi/filesystem/PinotFSFactory.java
b/pinot-spi/src/main/java/org/apache/pinot/spi/filesystem/PinotFSFactory.java
index 7003a77..eb5367c 100644
---
a/pinot-spi/src/main/java/org/apache/pinot/spi/filesystem/PinotFSFactory.java
+++
b/pinot-spi/src/main/java/org/apache/pinot/spi/filesystem/PinotFSFactory.java
@@ -60,19 +60,19 @@ public class PinotFSFactory {
}
}
- public static void init(PinotConfiguration fsConfig) {
+ public static void init(PinotConfiguration config) {
// Get schemes and their respective classes
-PinotConfiguration schemesConfiguration = fsConfig.subset(CLASS);
+PinotConfiguration schemesConfiguration = config.subset(CLASS);
List schemes = schemesConfiguration.getKeys();
if (!schemes.isEmpty()) {
LOGGER.info("Did not find any fs classes in the configuration");
}
for(String scheme : schemes){
- String fsClassName = (String) schemesConfiguration.getProperty(scheme);
-
- LOGGER.info("Got scheme {}, classname {}, starting to initialize",
scheme, fsClassName);
- register(scheme, fsClassName, schemesConfiguration.subset(scheme));
+ String fsClassName = schemesConfiguration.getProperty(scheme);
+ PinotConfiguration fsConfiguration = config.subset(scheme);
+ LOGGER.info("Got scheme {}, initializing class {} with config : {} ",
scheme, fsClassName, fsConfiguration.toMap());
+ register(scheme, fsClassName, fsConfiguration);
}
}
-
To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org
For additional commands, e-mail: commits-h...@pinot.apache.org
[incubator-pinot] branch pinot-fs-bug-fix created (now 2ec7dee)
This is an automated email from the ASF dual-hosted git repository. kishoreg pushed a change to branch pinot-fs-bug-fix in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git. at 2ec7dee Optimize selection order-by when not all selected expressions are ordered (#5661) No new revisions were added by this update. - To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
