This is an automated email from the ASF dual-hosted git repository.
jackie pushed a commit to branch fix_realtime_race_condition
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
The following commit(s) were added to refs/heads/fix_realtime_race_condition by
this push:
new 0a914da Handle multi-valued column
0a914da is described below
commit 0a914da71214b19f36c55fe25f9ca8d460d1c4cc
Author: Jackie (Xiaotian) Jiang <[email protected]>
AuthorDate: Tue Apr 2 11:13:54 2019 -0700
Handle multi-valued column
---
.../invertedindex/RealtimeInvertedIndexReader.java | 12 ++++--
.../RealtimeInvertedIndexReaderTest.java | 46 +++++++++++++++++-----
2 files changed, 45 insertions(+), 13 deletions(-)
diff --git
a/pinot-core/src/main/java/org/apache/pinot/core/realtime/impl/invertedindex/RealtimeInvertedIndexReader.java
b/pinot-core/src/main/java/org/apache/pinot/core/realtime/impl/invertedindex/RealtimeInvertedIndexReader.java
index 1250da0..28db4a1 100644
---
a/pinot-core/src/main/java/org/apache/pinot/core/realtime/impl/invertedindex/RealtimeInvertedIndexReader.java
+++
b/pinot-core/src/main/java/org/apache/pinot/core/realtime/impl/invertedindex/RealtimeInvertedIndexReader.java
@@ -25,6 +25,10 @@ import
org.apache.pinot.core.segment.index.readers.InvertedIndexReader;
import org.roaringbitmap.buffer.MutableRoaringBitmap;
+/**
+ * Real-time bitmap based inverted index reader which allows adding values on
the fly.
+ * <p>This class is thread-safe for single writer multiple readers.
+ */
public class RealtimeInvertedIndexReader implements
InvertedIndexReader<MutableRoaringBitmap> {
private final List<ThreadSafeMutableRoaringBitmap> _bitmaps = new
ArrayList<>();
private final ReentrantReadWriteLock.ReadLock _readLock;
@@ -37,7 +41,7 @@ public class RealtimeInvertedIndexReader implements
InvertedIndexReader<MutableR
}
/**
- * Add the document id to the bitmap for the given dictionary id.
+ * Adds the doc Id to the given dict Id's bitmap.
*/
public void add(int dictId, int docId) {
if (_bitmaps.size() == dictId) {
@@ -61,10 +65,10 @@ public class RealtimeInvertedIndexReader implements
InvertedIndexReader<MutableR
try {
_readLock.lock();
// NOTE: the given dict Id might not be added to the inverted index yet.
We first add the value to the dictionary.
- // Before the value is added to the inverted index, the query might have
a predicate which matches the newly added
+ // Before the value is added to the inverted index, the query might have
predicates that match the newly added
// value. In that case, the given dictionary Id does not exist in the
inverted index, and we return an empty
- // bitmap.
- if (_bitmaps.size() == dictId) {
+ // bitmap. For multi-valued column, the dict Id might be larger than the
bitmap size (not equal).
+ if (_bitmaps.size() <= dictId) {
return new MutableRoaringBitmap();
}
bitmap = _bitmaps.get(dictId);
diff --git
a/pinot-core/src/test/java/org/apache/pinot/core/realtime/impl/invertedindex/RealtimeInvertedIndexReaderTest.java
b/pinot-core/src/test/java/org/apache/pinot/core/realtime/impl/invertedindex/RealtimeInvertedIndexReaderTest.java
index 2191a23..7ecdc56 100644
---
a/pinot-core/src/test/java/org/apache/pinot/core/realtime/impl/invertedindex/RealtimeInvertedIndexReaderTest.java
+++
b/pinot-core/src/test/java/org/apache/pinot/core/realtime/impl/invertedindex/RealtimeInvertedIndexReaderTest.java
@@ -32,47 +32,75 @@ public class RealtimeInvertedIndexReaderTest {
public void testRealtimeInvertedIndexReader() {
RealtimeInvertedIndexReader realtimeInvertedIndexReader = new
RealtimeInvertedIndexReader();
- // Dict Id not added yet
+ // Add dict Id 0, doc Id 0 to the inverted index (single-value dict Id not
added yet)
+ // Before adding
MutableRoaringBitmap docIds = realtimeInvertedIndexReader.getDocIds(0);
assertNotNull(docIds);
assertTrue(docIds.isEmpty());
-
- // Add dict Id 0, doc Id 0 to the inverted index
+ // After adding
realtimeInvertedIndexReader.add(0, 0);
docIds = realtimeInvertedIndexReader.getDocIds(0);
assertNotNull(docIds);
assertFalse(docIds.isEmpty());
assertTrue(docIds.contains(0));
assertFalse(docIds.contains(1));
+
+ // Add dict Id 0, doc Id 1 to the inverted index (single-value dict Id
already added)
+ // Before adding
docIds = realtimeInvertedIndexReader.getDocIds(1);
assertNotNull(docIds);
assertTrue(docIds.isEmpty());
-
- // Add dict Id 0, doc Id 1 to the inverted index
realtimeInvertedIndexReader.add(0, 1);
+ // After adding
docIds = realtimeInvertedIndexReader.getDocIds(0);
assertNotNull(docIds);
assertFalse(docIds.isEmpty());
assertTrue(docIds.contains(0));
assertTrue(docIds.contains(1));
+
+ // Add dict Id 1 and 2, doc Id 2 to the inverted index (multi-value dict
Ids not added yet)
+ // Before adding dict Id 1
docIds = realtimeInvertedIndexReader.getDocIds(1);
assertNotNull(docIds);
assertTrue(docIds.isEmpty());
-
- // Add dict Id 1, doc Id 1 to the inverted index
- realtimeInvertedIndexReader.add(1, 1);
+ docIds = realtimeInvertedIndexReader.getDocIds(2);
+ assertNotNull(docIds);
+ assertTrue(docIds.isEmpty());
+ // After adding dict Id 1 but before adding dict Id 2
+ realtimeInvertedIndexReader.add(1, 2);
docIds = realtimeInvertedIndexReader.getDocIds(0);
assertNotNull(docIds);
assertFalse(docIds.isEmpty());
assertTrue(docIds.contains(0));
assertTrue(docIds.contains(1));
+ assertFalse(docIds.contains(2));
docIds = realtimeInvertedIndexReader.getDocIds(1);
assertNotNull(docIds);
assertFalse(docIds.isEmpty());
assertFalse(docIds.contains(0));
- assertTrue(docIds.contains(1));
+ assertFalse(docIds.contains(1));
+ assertTrue(docIds.contains(2));
docIds = realtimeInvertedIndexReader.getDocIds(2);
assertNotNull(docIds);
assertTrue(docIds.isEmpty());
+ // After adding dict Id 2
+ realtimeInvertedIndexReader.add(2, 2);
+ docIds = realtimeInvertedIndexReader.getDocIds(0);
+ assertNotNull(docIds);
+ assertFalse(docIds.isEmpty());
+ assertTrue(docIds.contains(0));
+ assertTrue(docIds.contains(1));
+ assertFalse(docIds.contains(2));
+ docIds = realtimeInvertedIndexReader.getDocIds(1);
+ assertNotNull(docIds);
+ assertFalse(docIds.isEmpty());
+ assertFalse(docIds.contains(0));
+ assertFalse(docIds.contains(1));
+ assertTrue(docIds.contains(2));
+ docIds = realtimeInvertedIndexReader.getDocIds(2);
+ assertFalse(docIds.isEmpty());
+ assertFalse(docIds.contains(0));
+ assertFalse(docIds.contains(1));
+ assertTrue(docIds.contains(2));
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
