This is an automated email from the ASF dual-hosted git repository. jamesshao pushed a commit to branch upsert in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
commit 11adb6e970bbb2230a9155931dff5b414aecce69 Author: james Shao <[email protected]> AuthorDate: Tue Nov 5 15:15:12 2019 -0800 change validFrom/validUntil column data type to int Summary: change validFrom/until data type from long to int, change memory usage per document from 16 bytes to 8 bytes Reviewers: bzzhang, csoman, tingchen Reviewed By: bzzhang Differential Revision: https://code.uberinternal.com/D3534725 --- pinot-api/pom.xml | 2 +- pinot-azure-filesystem/pom.xml | 2 +- pinot-broker/pom.xml | 2 +- pinot-common/pom.xml | 2 +- pinot-connectors/pinot-connector-kafka-0.11/pom.xml | 2 +- pinot-connectors/pinot-connector-kafka-0.9/pom.xml | 2 +- pinot-connectors/pom.xml | 2 +- pinot-controller/pom.xml | 2 +- pinot-core/pom.xml | 2 +- .../immutable/ImmutableUpsertSegmentImpl.java | 2 +- .../indexsegment/mutable/MutableUpsertSegmentImpl.java | 1 + .../mutable/ValidFromInMemoryVirtualColumnProvider.java | 7 ++++++- .../mutable/ValidUntilInMemoryVirtualColumnProvider.java | 7 ++++++- .../mutable/VirtualColumnLongValueReaderWriter.java | 16 +++++++++++----- pinot-distribution/pom.xml | 2 +- pinot-grigio/pinot-grigio-common/pom.xml | 2 +- .../retentionManager/UpdateLogTableRetentionManager.java | 3 ++- pinot-grigio/pinot-grigio-coordinator/pom.xml | 2 +- pinot-grigio/pom.xml | 2 +- pinot-hadoop-filesystem/pom.xml | 2 +- pinot-hadoop/pom.xml | 2 +- pinot-integration-tests/pom.xml | 2 +- pinot-minion/pom.xml | 2 +- pinot-orc/pom.xml | 2 +- pinot-parquet/pom.xml | 2 +- pinot-perf/pom.xml | 2 +- pinot-server/pom.xml | 2 +- pinot-tools/pom.xml | 2 +- pinot-transport/pom.xml | 2 +- pom.xml | 2 +- 30 files changed, 51 insertions(+), 33 deletions(-) diff --git a/pinot-api/pom.xml b/pinot-api/pom.xml index 6d6e96b..8763b4b 100644 --- a/pinot-api/pom.xml +++ b/pinot-api/pom.xml @@ -24,7 +24,7 @@ <parent> <artifactId>pinot</artifactId> <groupId>org.apache.pinot</groupId> - <version>0.2.2.5.45-SNAPSHOT</version> + <version>0.2.2.5.49-SNAPSHOT</version> </parent> <artifactId>pinot-api</artifactId> <name>Pinot API</name> diff --git a/pinot-azure-filesystem/pom.xml b/pinot-azure-filesystem/pom.xml index fe6c1c3..e42b355 100644 --- a/pinot-azure-filesystem/pom.xml +++ b/pinot-azure-filesystem/pom.xml @@ -24,7 +24,7 @@ <parent> <artifactId>pinot</artifactId> <groupId>org.apache.pinot</groupId> - <version>0.2.2.5.45-SNAPSHOT</version> + <version>0.2.2.5.49-SNAPSHOT</version> </parent> <artifactId>pinot-azure-filesystem</artifactId> <name>Pinot Azure Filesystem</name> diff --git a/pinot-broker/pom.xml b/pinot-broker/pom.xml index fbeb628..d625e2d 100644 --- a/pinot-broker/pom.xml +++ b/pinot-broker/pom.xml @@ -24,7 +24,7 @@ <parent> <artifactId>pinot</artifactId> <groupId>org.apache.pinot</groupId> - <version>0.2.2.5.45-SNAPSHOT</version> + <version>0.2.2.5.49-SNAPSHOT</version> </parent> <artifactId>pinot-broker</artifactId> <name>Pinot Broker</name> diff --git a/pinot-common/pom.xml b/pinot-common/pom.xml index 91df4fe..cde3078 100644 --- a/pinot-common/pom.xml +++ b/pinot-common/pom.xml @@ -24,7 +24,7 @@ <parent> <artifactId>pinot</artifactId> <groupId>org.apache.pinot</groupId> - <version>0.2.2.5.45-SNAPSHOT</version> + <version>0.2.2.5.49-SNAPSHOT</version> </parent> <artifactId>pinot-common</artifactId> <name>Pinot Common</name> diff --git a/pinot-connectors/pinot-connector-kafka-0.11/pom.xml b/pinot-connectors/pinot-connector-kafka-0.11/pom.xml index 50ef1ad..0b32fe0 100644 --- a/pinot-connectors/pinot-connector-kafka-0.11/pom.xml +++ b/pinot-connectors/pinot-connector-kafka-0.11/pom.xml @@ -23,7 +23,7 @@ <parent> <artifactId>pinot-connectors</artifactId> <groupId>org.apache.pinot</groupId> - <version>0.2.2.5.45-SNAPSHOT</version> + <version>0.2.2.5.49-SNAPSHOT</version> <relativePath>..</relativePath> </parent> <modelVersion>4.0.0</modelVersion> diff --git a/pinot-connectors/pinot-connector-kafka-0.9/pom.xml b/pinot-connectors/pinot-connector-kafka-0.9/pom.xml index 409e3cf..f409a8f 100644 --- a/pinot-connectors/pinot-connector-kafka-0.9/pom.xml +++ b/pinot-connectors/pinot-connector-kafka-0.9/pom.xml @@ -23,7 +23,7 @@ <parent> <artifactId>pinot-connectors</artifactId> <groupId>org.apache.pinot</groupId> - <version>0.2.2.5.45-SNAPSHOT</version> + <version>0.2.2.5.49-SNAPSHOT</version> <relativePath>..</relativePath> </parent> <modelVersion>4.0.0</modelVersion> diff --git a/pinot-connectors/pom.xml b/pinot-connectors/pom.xml index 749ec5c..b2cfd90 100644 --- a/pinot-connectors/pom.xml +++ b/pinot-connectors/pom.xml @@ -24,7 +24,7 @@ <parent> <artifactId>pinot</artifactId> <groupId>org.apache.pinot</groupId> - <version>0.2.2.5.45-SNAPSHOT</version> + <version>0.2.2.5.49-SNAPSHOT</version> </parent> <artifactId>pinot-connectors</artifactId> diff --git a/pinot-controller/pom.xml b/pinot-controller/pom.xml index 0e0af54..cf1e1f3 100644 --- a/pinot-controller/pom.xml +++ b/pinot-controller/pom.xml @@ -24,7 +24,7 @@ <parent> <artifactId>pinot</artifactId> <groupId>org.apache.pinot</groupId> - <version>0.2.2.5.45-SNAPSHOT</version> + <version>0.2.2.5.49-SNAPSHOT</version> </parent> <artifactId>pinot-controller</artifactId> <name>Pinot Controller</name> diff --git a/pinot-core/pom.xml b/pinot-core/pom.xml index a4e6ca8..141249d 100644 --- a/pinot-core/pom.xml +++ b/pinot-core/pom.xml @@ -24,7 +24,7 @@ <parent> <artifactId>pinot</artifactId> <groupId>org.apache.pinot</groupId> - <version>0.2.2.5.45-SNAPSHOT</version> + <version>0.2.2.5.49-SNAPSHOT</version> </parent> <artifactId>pinot-core</artifactId> <name>Pinot Core</name> diff --git a/pinot-core/src/main/java/org/apache/pinot/core/indexsegment/immutable/ImmutableUpsertSegmentImpl.java b/pinot-core/src/main/java/org/apache/pinot/core/indexsegment/immutable/ImmutableUpsertSegmentImpl.java index f42255f..80723b4 100644 --- a/pinot-core/src/main/java/org/apache/pinot/core/indexsegment/immutable/ImmutableUpsertSegmentImpl.java +++ b/pinot-core/src/main/java/org/apache/pinot/core/indexsegment/immutable/ImmutableUpsertSegmentImpl.java @@ -150,7 +150,7 @@ public class ImmutableUpsertSegmentImpl extends ImmutableSegmentImpl implements int docId = getDocIdFromSourceOffset(offset); StringBuilder result = new StringBuilder("matched: "); for (VirtualColumnLongValueReaderWriter readerWriter : _virtualColumnsReaderWriter) { - result.append(readerWriter.getLong(docId)).append("; "); + result.append(readerWriter.getInt(docId)).append("; "); } return result.toString(); } diff --git a/pinot-core/src/main/java/org/apache/pinot/core/indexsegment/mutable/MutableUpsertSegmentImpl.java b/pinot-core/src/main/java/org/apache/pinot/core/indexsegment/mutable/MutableUpsertSegmentImpl.java index b4d2e15..b69221d 100644 --- a/pinot-core/src/main/java/org/apache/pinot/core/indexsegment/mutable/MutableUpsertSegmentImpl.java +++ b/pinot-core/src/main/java/org/apache/pinot/core/indexsegment/mutable/MutableUpsertSegmentImpl.java @@ -44,6 +44,7 @@ public class MutableUpsertSegmentImpl extends MutableSegmentImpl implements Upse private final String _kafkaOffsetColumnName; + private final List<VirtualColumnLongValueReaderWriter> _mutableSegmentReaderWriters = new ArrayList<>(); // use map for mapping between kafka offset and docId because we at-most have 1 mutable segment per consumer // will use more memory, but we can update this later diff --git a/pinot-core/src/main/java/org/apache/pinot/core/segment/virtualcolumn/mutable/ValidFromInMemoryVirtualColumnProvider.java b/pinot-core/src/main/java/org/apache/pinot/core/segment/virtualcolumn/mutable/ValidFromInMemoryVirtualColumnProvider.java index c8ea4a1..2dedc8e 100644 --- a/pinot-core/src/main/java/org/apache/pinot/core/segment/virtualcolumn/mutable/ValidFromInMemoryVirtualColumnProvider.java +++ b/pinot-core/src/main/java/org/apache/pinot/core/segment/virtualcolumn/mutable/ValidFromInMemoryVirtualColumnProvider.java @@ -28,12 +28,17 @@ public class ValidFromInMemoryVirtualColumnProvider extends BaseLongVirtualColum public DataFileReader buildReader(VirtualColumnContext context) { return new VirtualColumnLongValueReaderWriter(context) { @Override - public boolean update(int docId, long value, LogEventType eventType) { + public boolean update(int docId, int value, LogEventType eventType) { if (eventType == LogEventType.INSERT) { return updateValue(docId, value); } return false; } + + @Override + public boolean update(int docId, long value, LogEventType eventType) { + return update(docId, Math.toIntExact(value), eventType); + } }; } } diff --git a/pinot-core/src/main/java/org/apache/pinot/core/segment/virtualcolumn/mutable/ValidUntilInMemoryVirtualColumnProvider.java b/pinot-core/src/main/java/org/apache/pinot/core/segment/virtualcolumn/mutable/ValidUntilInMemoryVirtualColumnProvider.java index e6e38c0..1a0856e 100644 --- a/pinot-core/src/main/java/org/apache/pinot/core/segment/virtualcolumn/mutable/ValidUntilInMemoryVirtualColumnProvider.java +++ b/pinot-core/src/main/java/org/apache/pinot/core/segment/virtualcolumn/mutable/ValidUntilInMemoryVirtualColumnProvider.java @@ -28,12 +28,17 @@ public class ValidUntilInMemoryVirtualColumnProvider extends BaseLongVirtualColu public DataFileReader buildReader(VirtualColumnContext context) { return new VirtualColumnLongValueReaderWriter(context) { @Override - public boolean update(int docId, long offset, LogEventType eventType) { + public boolean update(int docId, int offset, LogEventType eventType) { if (eventType == LogEventType.DELETE) { return this.updateValue(docId, offset); } return false; } + + @Override + public boolean update(int docId, long offset, LogEventType eventType) { + return update(docId, Math.toIntExact(offset), eventType); + } }; } } diff --git a/pinot-core/src/main/java/org/apache/pinot/core/segment/virtualcolumn/mutable/VirtualColumnLongValueReaderWriter.java b/pinot-core/src/main/java/org/apache/pinot/core/segment/virtualcolumn/mutable/VirtualColumnLongValueReaderWriter.java index e9df80d..75d1b74 100644 --- a/pinot-core/src/main/java/org/apache/pinot/core/segment/virtualcolumn/mutable/VirtualColumnLongValueReaderWriter.java +++ b/pinot-core/src/main/java/org/apache/pinot/core/segment/virtualcolumn/mutable/VirtualColumnLongValueReaderWriter.java @@ -30,18 +30,17 @@ import java.util.Arrays; public abstract class VirtualColumnLongValueReaderWriter extends BaseVirtualColumnSingleValueReaderWriter<ChunkReaderContext> { private static final Logger LOGGER = LoggerFactory.getLogger(VirtualColumnLongValueReaderWriter.class); - private static final long DEFAULT_VALUE = Long.MIN_VALUE; private final VirtualColumnContext _context; private int _totalDocSize; private int _currentMaxDocId; - private final long[] _values; - private final long DEFAULT_NEW_VALUE = -1; + private final int [] _values; + private final int DEFAULT_NEW_VALUE = -1; public VirtualColumnLongValueReaderWriter(VirtualColumnContext context) { _context = context; _totalDocSize = context.getTotalDocCount(); - _values = new long[_totalDocSize]; + _values = new int[_totalDocSize]; if (!_context.isMutableSegment()) { Arrays.fill(_values, -1); } @@ -84,7 +83,7 @@ public abstract class VirtualColumnLongValueReaderWriter extends BaseVirtualColu * method to update the internal data to value at a given location * synchronized to ensure the we will not modify the internal array at the same time from multiple threads */ - protected synchronized boolean updateValue(int docId, long value) { + protected synchronized boolean updateValue(int docId, int value) { if (docId >= _totalDocSize) { throw new RuntimeException(String.format("new record docId %s is larger than capacity %s", docId, _totalDocSize)); } @@ -100,5 +99,12 @@ public abstract class VirtualColumnLongValueReaderWriter extends BaseVirtualColu } } + // ensure backward compatibility + protected boolean updateValue(int docId, long value) { + return updateValue(docId, Math.toIntExact(value)); + } + public abstract boolean update(int docId, long value, LogEventType eventType); + + public abstract boolean update(int docId, int value, LogEventType eventType); } diff --git a/pinot-distribution/pom.xml b/pinot-distribution/pom.xml index fb652d0..ebf0146 100644 --- a/pinot-distribution/pom.xml +++ b/pinot-distribution/pom.xml @@ -25,7 +25,7 @@ <parent> <artifactId>pinot</artifactId> <groupId>org.apache.pinot</groupId> - <version>0.2.2.5.45-SNAPSHOT</version> + <version>0.2.2.5.49-SNAPSHOT</version> </parent> <artifactId>pinot-distribution</artifactId> <name>Pinot Distribution</name> diff --git a/pinot-grigio/pinot-grigio-common/pom.xml b/pinot-grigio/pinot-grigio-common/pom.xml index 6d9d4bb..42e07c0 100644 --- a/pinot-grigio/pinot-grigio-common/pom.xml +++ b/pinot-grigio/pinot-grigio-common/pom.xml @@ -23,7 +23,7 @@ <parent> <artifactId>pinot-grigio</artifactId> <groupId>org.apache.pinot</groupId> - <version>0.2.2.5.45-SNAPSHOT</version> + <version>0.2.2.5.49-SNAPSHOT</version> </parent> <modelVersion>4.0.0</modelVersion> diff --git a/pinot-grigio/pinot-grigio-common/src/main/java/org/apache/pinot/grigio/common/storageProvider/retentionManager/UpdateLogTableRetentionManager.java b/pinot-grigio/pinot-grigio-common/src/main/java/org/apache/pinot/grigio/common/storageProvider/retentionManager/UpdateLogTableRetentionManager.java index 62215b8..85efb47 100644 --- a/pinot-grigio/pinot-grigio-common/src/main/java/org/apache/pinot/grigio/common/storageProvider/retentionManager/UpdateLogTableRetentionManager.java +++ b/pinot-grigio/pinot-grigio-common/src/main/java/org/apache/pinot/grigio/common/storageProvider/retentionManager/UpdateLogTableRetentionManager.java @@ -78,7 +78,7 @@ public abstract class UpdateLogTableRetentionManager { int partition = llcSegmentName.getPartitionId(); if (!partitionLastSeg.containsKey(partition)) { partitionLastSeg.put(partition, llcSegmentName); - } else if (partitionLastSeg.get(partition).getSequenceNumber() < llcSegmentName.getSequenceNumber()) { + } else if (compareSegment(llcSegmentName, partitionLastSeg.get(partition))) { partitionLastSeg.put(partition, llcSegmentName); } } @@ -110,6 +110,7 @@ public abstract class UpdateLogTableRetentionManager { } // we most probably got a segment that is from a deleted table or segment assigned to another table // assume we don't do re-balance, we won't do refresh + LOGGER.warn("adding segment {} to blacklist", segmentName); _blacklistedSegments.put(segmentName, segmentName); return false; } diff --git a/pinot-grigio/pinot-grigio-coordinator/pom.xml b/pinot-grigio/pinot-grigio-coordinator/pom.xml index bc85f0f..19499d1 100644 --- a/pinot-grigio/pinot-grigio-coordinator/pom.xml +++ b/pinot-grigio/pinot-grigio-coordinator/pom.xml @@ -23,7 +23,7 @@ <parent> <artifactId>pinot-grigio</artifactId> <groupId>org.apache.pinot</groupId> - <version>0.2.2.5.45-SNAPSHOT</version> + <version>0.2.2.5.49-SNAPSHOT</version> </parent> <modelVersion>4.0.0</modelVersion> <artifactId>pinot-grigio-coordinator</artifactId> diff --git a/pinot-grigio/pom.xml b/pinot-grigio/pom.xml index 2766d3a..3cdad0c 100644 --- a/pinot-grigio/pom.xml +++ b/pinot-grigio/pom.xml @@ -23,7 +23,7 @@ <parent> <artifactId>pinot</artifactId> <groupId>org.apache.pinot</groupId> - <version>0.2.2.5.45-SNAPSHOT</version> + <version>0.2.2.5.49-SNAPSHOT</version> </parent> <modelVersion>4.0.0</modelVersion> <artifactId>pinot-grigio</artifactId> diff --git a/pinot-hadoop-filesystem/pom.xml b/pinot-hadoop-filesystem/pom.xml index 6d44587..49b0a8a 100644 --- a/pinot-hadoop-filesystem/pom.xml +++ b/pinot-hadoop-filesystem/pom.xml @@ -24,7 +24,7 @@ <parent> <artifactId>pinot</artifactId> <groupId>org.apache.pinot</groupId> - <version>0.2.2.5.45-SNAPSHOT</version> + <version>0.2.2.5.49-SNAPSHOT</version> </parent> <artifactId>pinot-hadoop-filesystem</artifactId> <name>Pinot Hadoop Filesystem</name> diff --git a/pinot-hadoop/pom.xml b/pinot-hadoop/pom.xml index 2272b24..928a11b 100644 --- a/pinot-hadoop/pom.xml +++ b/pinot-hadoop/pom.xml @@ -24,7 +24,7 @@ <parent> <artifactId>pinot</artifactId> <groupId>org.apache.pinot</groupId> - <version>0.2.2.5.45-SNAPSHOT</version> + <version>0.2.2.5.49-SNAPSHOT</version> </parent> <artifactId>pinot-hadoop</artifactId> <name>Pinot Hadoop</name> diff --git a/pinot-integration-tests/pom.xml b/pinot-integration-tests/pom.xml index 1095763..9d7cb12 100644 --- a/pinot-integration-tests/pom.xml +++ b/pinot-integration-tests/pom.xml @@ -24,7 +24,7 @@ <parent> <artifactId>pinot</artifactId> <groupId>org.apache.pinot</groupId> - <version>0.2.2.5.45-SNAPSHOT</version> + <version>0.2.2.5.49-SNAPSHOT</version> </parent> <artifactId>pinot-integration-tests</artifactId> <name>Pinot Integration Tests</name> diff --git a/pinot-minion/pom.xml b/pinot-minion/pom.xml index a7352ee..c057094 100644 --- a/pinot-minion/pom.xml +++ b/pinot-minion/pom.xml @@ -24,7 +24,7 @@ <parent> <artifactId>pinot</artifactId> <groupId>org.apache.pinot</groupId> - <version>0.2.2.5.45-SNAPSHOT</version> + <version>0.2.2.5.49-SNAPSHOT</version> </parent> <artifactId>pinot-minion</artifactId> <name>Pinot Minion</name> diff --git a/pinot-orc/pom.xml b/pinot-orc/pom.xml index e046ca8..34a8740 100644 --- a/pinot-orc/pom.xml +++ b/pinot-orc/pom.xml @@ -23,7 +23,7 @@ <parent> <artifactId>pinot</artifactId> <groupId>org.apache.pinot</groupId> - <version>0.2.2.5.45-SNAPSHOT</version> + <version>0.2.2.5.49-SNAPSHOT</version> </parent> <modelVersion>4.0.0</modelVersion> diff --git a/pinot-parquet/pom.xml b/pinot-parquet/pom.xml index 5c5b9b0..2d7dbb0 100644 --- a/pinot-parquet/pom.xml +++ b/pinot-parquet/pom.xml @@ -23,7 +23,7 @@ <parent> <artifactId>pinot</artifactId> <groupId>org.apache.pinot</groupId> - <version>0.2.2.5.45-SNAPSHOT</version> + <version>0.2.2.5.49-SNAPSHOT</version> </parent> <modelVersion>4.0.0</modelVersion> diff --git a/pinot-perf/pom.xml b/pinot-perf/pom.xml index 3312026..060ebaf 100644 --- a/pinot-perf/pom.xml +++ b/pinot-perf/pom.xml @@ -24,7 +24,7 @@ <parent> <artifactId>pinot</artifactId> <groupId>org.apache.pinot</groupId> - <version>0.2.2.5.45-SNAPSHOT</version> + <version>0.2.2.5.49-SNAPSHOT</version> </parent> <artifactId>pinot-perf</artifactId> <name>Pinot Perf</name> diff --git a/pinot-server/pom.xml b/pinot-server/pom.xml index 9463c64..ac77f20 100644 --- a/pinot-server/pom.xml +++ b/pinot-server/pom.xml @@ -24,7 +24,7 @@ <parent> <artifactId>pinot</artifactId> <groupId>org.apache.pinot</groupId> - <version>0.2.2.5.45-SNAPSHOT</version> + <version>0.2.2.5.49-SNAPSHOT</version> </parent> <artifactId>pinot-server</artifactId> <name>Pinot Server</name> diff --git a/pinot-tools/pom.xml b/pinot-tools/pom.xml index 7aa523f..4b4e686 100644 --- a/pinot-tools/pom.xml +++ b/pinot-tools/pom.xml @@ -24,7 +24,7 @@ <parent> <artifactId>pinot</artifactId> <groupId>org.apache.pinot</groupId> - <version>0.2.2.5.45-SNAPSHOT</version> + <version>0.2.2.5.49-SNAPSHOT</version> </parent> <artifactId>pinot-tools</artifactId> <name>Pinot Tools</name> diff --git a/pinot-transport/pom.xml b/pinot-transport/pom.xml index dc748c7..b977921 100644 --- a/pinot-transport/pom.xml +++ b/pinot-transport/pom.xml @@ -24,7 +24,7 @@ <parent> <artifactId>pinot</artifactId> <groupId>org.apache.pinot</groupId> - <version>0.2.2.5.45-SNAPSHOT</version> + <version>0.2.2.5.49-SNAPSHOT</version> </parent> <artifactId>pinot-transport</artifactId> <name>Pinot Transport</name> diff --git a/pom.xml b/pom.xml index c3d0cf0..77e8647 100644 --- a/pom.xml +++ b/pom.xml @@ -30,7 +30,7 @@ <groupId>org.apache.pinot</groupId> <artifactId>pinot</artifactId> - <version>0.2.2.5.45-SNAPSHOT</version> + <version>0.2.2.5.49-SNAPSHOT</version> <packaging>pom</packaging> <name>Pinot</name> <description>A realtime distributed OLAP datastore</description> --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
