mcvsubbu closed pull request #3512: Let PinotDataBuffer.newIndexFor(...) takes
long value
URL: https://github.com/apache/incubator-pinot/pull/3512
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git
a/pinot-core/src/main/java/com/linkedin/pinot/core/segment/index/converter/SegmentV1V2ToV3FormatConverter.java
b/pinot-core/src/main/java/com/linkedin/pinot/core/segment/index/converter/SegmentV1V2ToV3FormatConverter.java
index 833059c13f..da1b23cb87 100644
---
a/pinot-core/src/main/java/com/linkedin/pinot/core/segment/index/converter/SegmentV1V2ToV3FormatConverter.java
+++
b/pinot-core/src/main/java/com/linkedin/pinot/core/segment/index/converter/SegmentV1V2ToV3FormatConverter.java
@@ -198,7 +198,7 @@ private void
copyExistingInvertedIndex(SegmentDirectory.Reader reader,
private void readCopyBuffers(SegmentDirectory.Reader reader,
SegmentDirectory.Writer writer, String column,
ColumnIndexType indexType) throws IOException {
PinotDataBuffer oldBuffer = reader.getIndexFor(column, indexType);
- PinotDataBuffer newDictBuffer = writer.newIndexFor(column, indexType,
(int) oldBuffer.size());
+ PinotDataBuffer newDictBuffer = writer.newIndexFor(column, indexType,
oldBuffer.size());
oldBuffer.copyTo(0, newDictBuffer, 0, oldBuffer.size());
}
diff --git
a/pinot-core/src/main/java/com/linkedin/pinot/core/segment/index/loader/LoaderUtils.java
b/pinot-core/src/main/java/com/linkedin/pinot/core/segment/index/loader/LoaderUtils.java
index 2ba1a4d968..b8c30e48f1 100644
---
a/pinot-core/src/main/java/com/linkedin/pinot/core/segment/index/loader/LoaderUtils.java
+++
b/pinot-core/src/main/java/com/linkedin/pinot/core/segment/index/loader/LoaderUtils.java
@@ -48,7 +48,7 @@ private LoaderUtils() {
*/
public static void writeIndexToV3Format(SegmentDirectory.Writer
segmentWriter, String column, File indexFile,
ColumnIndexType indexType) throws IOException {
- int fileLength = (int) indexFile.length();
+ long fileLength = indexFile.length();
try (PinotDataBuffer buffer = segmentWriter.newIndexFor(column, indexType,
fileLength)) {
buffer.readFrom(0, indexFile, 0, fileLength);
FileUtils.forceDelete(indexFile);
diff --git
a/pinot-core/src/main/java/com/linkedin/pinot/core/segment/store/ColumnIndexDirectory.java
b/pinot-core/src/main/java/com/linkedin/pinot/core/segment/store/ColumnIndexDirectory.java
index c20a091ebb..3579f1c175 100644
---
a/pinot-core/src/main/java/com/linkedin/pinot/core/segment/store/ColumnIndexDirectory.java
+++
b/pinot-core/src/main/java/com/linkedin/pinot/core/segment/store/ColumnIndexDirectory.java
@@ -101,7 +101,7 @@ public abstract PinotDataBuffer
getInvertedIndexBufferFor(String column)
* @return in-memory ByteBuffer like buffer for data
* @throws IOException
*/
- public abstract PinotDataBuffer newDictionaryBuffer(String column, int
sizeBytes)
+ public abstract PinotDataBuffer newDictionaryBuffer(String column, long
sizeBytes)
throws IOException;
/**
* Allocate a new data buffer of specified sizeBytes in the columnar index
directory
@@ -110,7 +110,7 @@ public abstract PinotDataBuffer newDictionaryBuffer(String
column, int sizeBytes
* @return in-memory ByteBuffer like buffer for data
* @throws IOException
*/
- public abstract PinotDataBuffer newForwardIndexBuffer(String column, int
sizeBytes)
+ public abstract PinotDataBuffer newForwardIndexBuffer(String column, long
sizeBytes)
throws IOException;
/**
* Allocate a new data buffer of specified sizeBytes in the columnar index
directory
@@ -119,7 +119,7 @@ public abstract PinotDataBuffer
newForwardIndexBuffer(String column, int sizeByt
* @return in-memory ByteBuffer like buffer for data
* @throws IOException
*/
- public abstract PinotDataBuffer newInvertedIndexBuffer(String column, int
sizeBytes)
+ public abstract PinotDataBuffer newInvertedIndexBuffer(String column, long
sizeBytes)
throws IOException;
/**
diff --git
a/pinot-core/src/main/java/com/linkedin/pinot/core/segment/store/FilePerIndexDirectory.java
b/pinot-core/src/main/java/com/linkedin/pinot/core/segment/store/FilePerIndexDirectory.java
index 6c99582969..c6ea2d90c3 100644
---
a/pinot-core/src/main/java/com/linkedin/pinot/core/segment/store/FilePerIndexDirectory.java
+++
b/pinot-core/src/main/java/com/linkedin/pinot/core/segment/store/FilePerIndexDirectory.java
@@ -46,7 +46,7 @@ public PinotDataBuffer getDictionaryBufferFor(String column)
}
@Override
- public PinotDataBuffer newDictionaryBuffer(String column, int sizeBytes)
+ public PinotDataBuffer newDictionaryBuffer(String column, long sizeBytes)
throws IOException {
IndexKey key = new IndexKey(column, ColumnIndexType.DICTIONARY);
return getWriteBufferFor(key, sizeBytes);
@@ -60,7 +60,7 @@ public PinotDataBuffer getForwardIndexBufferFor(String column)
}
@Override
- public PinotDataBuffer newForwardIndexBuffer(String column, int sizeBytes)
+ public PinotDataBuffer newForwardIndexBuffer(String column, long sizeBytes)
throws IOException {
IndexKey key = new IndexKey(column, ColumnIndexType.FORWARD_INDEX);
return getWriteBufferFor(key, sizeBytes);
@@ -74,7 +74,7 @@ public PinotDataBuffer getInvertedIndexBufferFor(String
column)
}
@Override
- public PinotDataBuffer newInvertedIndexBuffer(String column, int sizeBytes)
+ public PinotDataBuffer newInvertedIndexBuffer(String column, long sizeBytes)
throws IOException {
IndexKey key = new IndexKey(column, ColumnIndexType.INVERTED_INDEX);
return getWriteBufferFor(key, sizeBytes);
@@ -116,7 +116,7 @@ private PinotDataBuffer getReadBufferFor(IndexKey key)
throws IOException {
return buffer;
}
- private PinotDataBuffer getWriteBufferFor(IndexKey key, int sizeBytes)
throws IOException {
+ private PinotDataBuffer getWriteBufferFor(IndexKey key, long sizeBytes)
throws IOException {
if (indexBuffers.containsKey(key)) {
return indexBuffers.get(key);
}
@@ -146,7 +146,7 @@ File getFileFor(String column, ColumnIndexType indexType) {
return new File(segmentDirectory, filename);
}
- private PinotDataBuffer mapForWrites(File file, int sizeBytes, String
context) throws IOException {
+ private PinotDataBuffer mapForWrites(File file, long sizeBytes, String
context) throws IOException {
Preconditions.checkNotNull(file);
Preconditions.checkArgument(sizeBytes >= 0 && sizeBytes <
Integer.MAX_VALUE,
"File size must be less than 2GB, file: " + file);
diff --git
a/pinot-core/src/main/java/com/linkedin/pinot/core/segment/store/SegmentDirectory.java
b/pinot-core/src/main/java/com/linkedin/pinot/core/segment/store/SegmentDirectory.java
index 8881ea59f7..918e61f5a4 100644
---
a/pinot-core/src/main/java/com/linkedin/pinot/core/segment/store/SegmentDirectory.java
+++
b/pinot-core/src/main/java/com/linkedin/pinot/core/segment/store/SegmentDirectory.java
@@ -181,7 +181,7 @@ public abstract PinotDataBuffer getIndexFor(String column,
ColumnIndexType type)
// NOTE: an interface like readFrom(File f, String column,
ColumnIndexType, int sizeBytes) will be safe
// but it can lead to potential endianness issues. Endianness used to
create data may not be
// same as PinotDataBufferOld
- public abstract PinotDataBuffer newIndexFor(String columnName,
ColumnIndexType indexType, int sizeBytes)
+ public abstract PinotDataBuffer newIndexFor(String columnName,
ColumnIndexType indexType, long sizeBytes)
throws IOException;
/**
diff --git
a/pinot-core/src/main/java/com/linkedin/pinot/core/segment/store/SegmentLocalFSDirectory.java
b/pinot-core/src/main/java/com/linkedin/pinot/core/segment/store/SegmentLocalFSDirectory.java
index 37cb57fd71..d29c37c295 100644
---
a/pinot-core/src/main/java/com/linkedin/pinot/core/segment/store/SegmentLocalFSDirectory.java
+++
b/pinot-core/src/main/java/com/linkedin/pinot/core/segment/store/SegmentLocalFSDirectory.java
@@ -356,7 +356,7 @@ public Writer() {
}
@Override
- public PinotDataBuffer newIndexFor(String columnName, ColumnIndexType
indexType, int sizeBytes)
+ public PinotDataBuffer newIndexFor(String columnName, ColumnIndexType
indexType, long sizeBytes)
throws IOException {
return getNewIndexBuffer(new IndexKey(columnName, indexType), sizeBytes);
}
@@ -407,12 +407,11 @@ private PinotDataBuffer getNewIndexBuffer(IndexKey key,
long sizeBytes)
ColumnIndexType indexType = key.type;
switch (indexType) {
case DICTIONARY:
- return columnIndexDirectory.newDictionaryBuffer(key.name, (int)
sizeBytes);
-
+ return columnIndexDirectory.newDictionaryBuffer(key.name, sizeBytes);
case FORWARD_INDEX:
- return columnIndexDirectory.newForwardIndexBuffer(key.name, (int)
sizeBytes);
+ return columnIndexDirectory.newForwardIndexBuffer(key.name,
sizeBytes);
case INVERTED_INDEX:
- return columnIndexDirectory.newInvertedIndexBuffer(key.name, ((int)
sizeBytes));
+ return columnIndexDirectory.newInvertedIndexBuffer(key.name,
sizeBytes);
default:
throw new RuntimeException("Unknown index type: " + indexType.name()
+
" for directory: " + segmentDirectory);
diff --git
a/pinot-core/src/main/java/com/linkedin/pinot/core/segment/store/SingleFileIndexDirectory.java
b/pinot-core/src/main/java/com/linkedin/pinot/core/segment/store/SingleFileIndexDirectory.java
index 127f437c0f..cf22cc64ce 100644
---
a/pinot-core/src/main/java/com/linkedin/pinot/core/segment/store/SingleFileIndexDirectory.java
+++
b/pinot-core/src/main/java/com/linkedin/pinot/core/segment/store/SingleFileIndexDirectory.java
@@ -111,19 +111,19 @@ public boolean hasIndexFor(String column, ColumnIndexType
type) {
}
@Override
- public PinotDataBuffer newDictionaryBuffer(String column, int sizeBytes)
+ public PinotDataBuffer newDictionaryBuffer(String column, long sizeBytes)
throws IOException {
return allocNewBufferInternal(column, ColumnIndexType.DICTIONARY,
sizeBytes, "dictionary.create");
}
@Override
- public PinotDataBuffer newForwardIndexBuffer(String column, int sizeBytes)
+ public PinotDataBuffer newForwardIndexBuffer(String column, long sizeBytes)
throws IOException {
return allocNewBufferInternal(column, ColumnIndexType.FORWARD_INDEX,
sizeBytes, "forward_index.create");
}
@Override
- public PinotDataBuffer newInvertedIndexBuffer(String column, int sizeBytes)
+ public PinotDataBuffer newInvertedIndexBuffer(String column, long sizeBytes)
throws IOException {
return allocNewBufferInternal(column, ColumnIndexType.INVERTED_INDEX,
sizeBytes, "inverted_index.create");
}
@@ -139,7 +139,7 @@ private PinotDataBuffer checkAndGetIndexBuffer(String
column, ColumnIndexType ty
}
// This is using extra resources right now which can be changed.
- private PinotDataBuffer allocNewBufferInternal(String column,
ColumnIndexType indexType, int size,
+ private PinotDataBuffer allocNewBufferInternal(String column,
ColumnIndexType indexType, long size,
String context)
throws IOException {
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
