This is an automated email from the ASF dual-hosted git repository.
snlee pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
The following commit(s) were added to refs/heads/master by this push:
new e5bf05b Fix the overflow issue when loading the large dictionary into
the buffer (#6476)
e5bf05b is described below
commit e5bf05b0ff831f8654fd1386a76b759266c6f8c7
Author: Seunghyun Lee <[email protected]>
AuthorDate: Mon Jan 25 12:37:51 2021 -0800
Fix the overflow issue when loading the large dictionary into the buffer
(#6476)
Currently, we allow to generate the large dictionary (>2GB) while loading
segment fails due to overflow. This pr fixes the issue.
---
.../apache/pinot/core/segment/store/SingleFileIndexDirectory.java | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git
a/pinot-core/src/main/java/org/apache/pinot/core/segment/store/SingleFileIndexDirectory.java
b/pinot-core/src/main/java/org/apache/pinot/core/segment/store/SingleFileIndexDirectory.java
index 6ba81f8..c34ad2f 100644
---
a/pinot-core/src/main/java/org/apache/pinot/core/segment/store/SingleFileIndexDirectory.java
+++
b/pinot-core/src/main/java/org/apache/pinot/core/segment/store/SingleFileIndexDirectory.java
@@ -180,7 +180,7 @@ class SingleFileIndexDirectory extends ColumnIndexDirectory
{
}
}
- private void validateMagicMarker(PinotDataBuffer buffer, int startOffset) {
+ private void validateMagicMarker(PinotDataBuffer buffer, long startOffset) {
long actualMarkerValue = buffer.getLong(startOffset);
if (actualMarkerValue != MAGIC_MARKER) {
LOGGER.error("Missing magic marker in index file: {} at position: {}",
indexFile, startOffset);
@@ -292,10 +292,10 @@ class SingleFileIndexDirectory extends
ColumnIndexDirectory {
}
allocBuffers.add(buffer);
- int prevSlicePoint = 0;
+ long prevSlicePoint = 0;
for (Long fileOffset : offsetAccum) {
IndexEntry entry = startOffsets.get(fileOffset);
- int endSlicePoint = prevSlicePoint + (int) entry.size;
+ long endSlicePoint = prevSlicePoint + entry.size;
validateMagicMarker(buffer, prevSlicePoint);
PinotDataBuffer viewBuffer = buffer.view(prevSlicePoint +
MAGIC_MARKER_SIZE_BYTES, endSlicePoint);
entry.buffer = viewBuffer;
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
