This is an automated email from the ASF dual-hosted git repository.
jamesshao pushed a change to branch upsert
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git.
from c10ab01 fix broker bug where upsert table is not queriable
new f35f1d7 [maven-release-plugin] prepare release pinot-0.2.2.5.43
new df8e3a8 [maven-release-plugin] prepare for next development iteration
new 5d46d78 Clean up imports for grigio
new 70a913c Implement FixedPartitionCountPartitioner
new 9ac12f3 fix issue in retention management and refactor
new 6df85f3 change offset mapping strategy for immutable seg to reduce
memory
new 11adb6e change validFrom/validUntil column data type to int
new fea13a0 Add retries to RocksDB reads and writes
new 4708139 Fix FixedPartitionCountBytesPartitionerTest
new 7c8cc38 improve pinot server upsert segment loading time
new 51684f7 small improve on loading data for virtual column
new 5f33a29 add one more lag metrics to kc
The 12 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
pinot-api/pom.xml | 2 +-
pinot-azure-filesystem/pom.xml | 2 +-
pinot-broker/pom.xml | 2 +-
pinot-common/pom.xml | 2 +-
.../pinot-connector-kafka-0.11/pom.xml | 2 +-
pinot-connectors/pinot-connector-kafka-0.9/pom.xml | 2 +-
pinot-connectors/pom.xml | 2 +-
pinot-controller/pom.xml | 2 +-
pinot-core/pom.xml | 3 +-
.../pinot/core/indexsegment/UpsertSegment.java | 3 +-
.../immutable/ImmutableUpsertSegmentImpl.java | 187 ++++++++++++++-------
.../mutable/MutableUpsertSegmentImpl.java | 9 +-
.../segment/updater/UpsertWaterMarkManager.java | 13 +-
.../ValidFromInMemoryVirtualColumnProvider.java | 10 +-
.../ValidUntilInMemoryVirtualColumnProvider.java | 10 +-
.../VirtualColumnLongValueReaderWriter.java | 38 +++--
.../immutable/ImmutableUpsertSegmentImplTest.java | 126 ++++++++++++++
pinot-distribution/pom.xml | 2 +-
pinot-grigio/pinot-grigio-common/pom.xml | 8 +-
.../common/keyValueStore/KeyValueStoreTable.java | 2 -
.../pinot/grigio/common/messages/LogEventType.java | 8 +-
.../pinot/grigio/common/metrics/GrigioGauge.java | 1 +
.../pinot/grigio/common/metrics/GrigioMeter.java | 1 -
.../common/rpcQueue/QueueConsumerRecord.java | 8 +-
.../SegmentUpdateLogStorageProvider.java | 26 +--
.../common/storageProvider/UpdateLogEntry.java | 3 +
.../common/storageProvider/UpdateLogEntrySet.java | 69 ++++++++
.../storageProvider/UpdateLogStorageExplorer.java | 7 +-
.../storageProvider/UpdateLogStorageProvider.java | 8 +-
.../UpdateLogRetentionManagerImpl.java | 2 +-
.../UpdateLogTableRetentionManager.java | 99 ++++++++---
.../UpdateLogTableRetentionManagerImpl.java | 79 +--------
.../grigio}/common/utils/IdealStateHelper.java | 2 +-
.../common/rpcQueue/QueueConsumerRecordTest.java | 3 +-
.../SegmentUpdateLogStorageProviderTest.java | 73 ++++++--
.../UpdateLogTableRetentionManagerImplTest.java | 119 +++++++++++++
.../grigio/common/utils/IdealStateHelperTest.java | 74 ++++++++
pinot-grigio/pinot-grigio-coordinator/pom.xml | 38 +++--
...va => FixedPartitionCountBytesPartitioner.java} | 36 ++--
...java => FixedPartitionCountIntPartitioner.java} | 30 ++--
...er.java => FixedPartitionCountPartitioner.java} | 25 +--
.../common/keyValueStore/RocksDBBatchReader.java | 62 +++++++
.../common/keyValueStore/RocksDBBatchWriter.java | 42 +++--
.../keyValueStore/RocksDBKeyValueStoreTable.java | 31 +---
.../grigio/common/rpcQueue/KafkaQueueConsumer.java | 3 +-
.../rpcQueue/KeyCoordinatorQueueProducer.java | 4 +-
.../rpcQueue/LogCoordinatorQueueProducer.java | 4 +-
.../common/rpcQueue/VersionMsgQueueProducer.java | 4 +-
.../KCUpdateLogRetentionManagerImpl.java | 2 +-
.../KCUpdateLogTableRetentionManagerImpl.java | 58 ++-----
.../internal/DistributedKeyCoordinatorCore.java | 2 +
.../keyCoordinator/starter/KeyCoordinatorConf.java | 2 +-
.../starter/KeyCoordinatorStarter.java | 2 +-
.../FixedPartitionCountBytesPartitionerTest.java | 75 +++++++++
.../FixedPartitionCountIntPartitionerTest.java | 74 ++++++++
.../KCUpdateLogRetentionManagerImplTest.java | 114 +++++++++++++
.../KCUpdateLogTableRetentionManagerImplTest.java | 148 ++++++++++++++++
.../org.mockito.plugins.MockMaker | 1 +
pinot-grigio/pom.xml | 2 +-
pinot-hadoop-filesystem/pom.xml | 2 +-
pinot-hadoop/pom.xml | 2 +-
pinot-integration-tests/pom.xml | 2 +-
pinot-minion/pom.xml | 2 +-
pinot-orc/pom.xml | 2 +-
pinot-parquet/pom.xml | 2 +-
pinot-perf/pom.xml | 2 +-
pinot-server/pom.xml | 3 +-
.../pinot/server/starter/ServerInstance.java | 2 +-
pinot-tools/pom.xml | 2 +-
pinot-transport/pom.xml | 2 +-
pom.xml | 7 +-
71 files changed, 1362 insertions(+), 436 deletions(-)
create mode 100644
pinot-core/src/test/java/org/apache/pinot/core/indexsegment/immutable/ImmutableUpsertSegmentImplTest.java
create mode 100644
pinot-grigio/pinot-grigio-common/src/main/java/org/apache/pinot/grigio/common/storageProvider/UpdateLogEntrySet.java
rename {pinot-common/src/main/java/org/apache/pinot =>
pinot-grigio/pinot-grigio-common/src/main/java/org/apache/pinot/grigio}/common/utils/IdealStateHelper.java
(98%)
create mode 100644
pinot-grigio/pinot-grigio-common/src/test/java/org/apache/pinot/grigio/common/storageProvider/retentionManager/UpdateLogTableRetentionManagerImplTest.java
create mode 100644
pinot-grigio/pinot-grigio-common/src/test/java/org/apache/pinot/grigio/common/utils/IdealStateHelperTest.java
copy
pinot-grigio/pinot-grigio-coordinator/src/main/java/org/apache/pinot/grigio/common/{IntPartitioner.java
=> FixedPartitionCountBytesPartitioner.java} (56%)
copy
pinot-grigio/pinot-grigio-coordinator/src/main/java/org/apache/pinot/grigio/common/{IntPartitioner.java
=> FixedPartitionCountIntPartitioner.java} (64%)
rename
pinot-grigio/pinot-grigio-coordinator/src/main/java/org/apache/pinot/grigio/common/{IntPartitioner.java
=> FixedPartitionCountPartitioner.java} (65%)
create mode 100644
pinot-grigio/pinot-grigio-coordinator/src/main/java/org/apache/pinot/grigio/common/keyValueStore/RocksDBBatchReader.java
copy
pinot-common/src/main/java/org/apache/pinot/common/restlet/resources/ResourceUtils.java
=>
pinot-grigio/pinot-grigio-coordinator/src/main/java/org/apache/pinot/grigio/common/keyValueStore/RocksDBBatchWriter.java
(51%)
create mode 100644
pinot-grigio/pinot-grigio-coordinator/src/test/java/org/apache/pinot/grigio/common/FixedPartitionCountBytesPartitionerTest.java
create mode 100644
pinot-grigio/pinot-grigio-coordinator/src/test/java/org/apache/pinot/grigio/common/FixedPartitionCountIntPartitionerTest.java
create mode 100644
pinot-grigio/pinot-grigio-coordinator/src/test/java/org/apache/pinot/grigio/common/storageProvider/retentionManager/KCUpdateLogRetentionManagerImplTest.java
create mode 100644
pinot-grigio/pinot-grigio-coordinator/src/test/java/org/apache/pinot/grigio/common/storageProvider/retentionManager/KCUpdateLogTableRetentionManagerImplTest.java
create mode 100644
pinot-grigio/pinot-grigio-coordinator/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
