This is an automated email from the ASF dual-hosted git repository. sunithabeeram pushed a commit to branch trackBytesRead in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
commit 5e1ac4ba02219f99a7550aa2dfc6a0bca2c2c779 Author: Sunitha Beeram <[email protected]> AuthorDate: Fri Dec 21 16:22:13 2018 -0800 [PINOT-7370] Return number of bytes read from the reader interfaces/implementations --- .../pinot/core/common/BaseBlockValSet.java | 12 +++--- .../linkedin/pinot/core/common/BlockValSet.java | 19 +++------- .../core/operator/docvalsets/SingleValueSet.java | 44 +++++++++++++--------- .../core/segment/index/readers/BaseDictionary.java | 28 ++++++++++---- .../segment/index/readers/BytesDictionary.java | 8 +++- .../core/segment/index/readers/Dictionary.java | 12 +++--- .../segment/index/readers/StringDictionary.java | 8 +++- 7 files changed, 77 insertions(+), 54 deletions(-) diff --git a/pinot-core/src/main/java/com/linkedin/pinot/core/common/BaseBlockValSet.java b/pinot-core/src/main/java/com/linkedin/pinot/core/common/BaseBlockValSet.java index f442755..f7a42ed 100644 --- a/pinot-core/src/main/java/com/linkedin/pinot/core/common/BaseBlockValSet.java +++ b/pinot-core/src/main/java/com/linkedin/pinot/core/common/BaseBlockValSet.java @@ -39,32 +39,32 @@ public abstract class BaseBlockValSet implements BlockValSet { } @Override - public void getIntValues(int[] inDocIds, int inStartPos, int inDocIdsSize, int[] outValues, int outStartPos) { + public long getIntValues(int[] inDocIds, int inStartPos, int inDocIdsSize, int[] outValues, int outStartPos) { throw new UnsupportedOperationException(); } @Override - public void getLongValues(int[] inDocIds, int inStartPos, int inDocIdsSize, long[] outValues, int outStartPos) { + public long getLongValues(int[] inDocIds, int inStartPos, int inDocIdsSize, long[] outValues, int outStartPos) { throw new UnsupportedOperationException(); } @Override - public void getFloatValues(int[] inDocIds, int inStartPos, int inDocIdsSize, float[] outValues, int outStartPos) { + public long getFloatValues(int[] inDocIds, int inStartPos, int inDocIdsSize, float[] outValues, int outStartPos) { throw new UnsupportedOperationException(); } @Override - public void getDoubleValues(int[] inDocIds, int inStartPos, int inDocIdsSize, double[] outValues, int outStartPos) { + public long getDoubleValues(int[] inDocIds, int inStartPos, int inDocIdsSize, double[] outValues, int outStartPos) { throw new UnsupportedOperationException(); } @Override - public void getStringValues(int[] inDocIds, int inStartPos, int inDocIdsSize, String[] outValues, int outStartPos) { + public long getStringValues(int[] inDocIds, int inStartPos, int inDocIdsSize, String[] outValues, int outStartPos) { throw new UnsupportedOperationException(); } @Override - public void getBytesValues(int[] inDocIds, int inStartPos, int inDocIdsSize, byte[][] outValues, int outStartPos) { + public long getBytesValues(int[] inDocIds, int inStartPos, int inDocIdsSize, byte[][] outValues, int outStartPos) { throw new UnsupportedOperationException(); } diff --git a/pinot-core/src/main/java/com/linkedin/pinot/core/common/BlockValSet.java b/pinot-core/src/main/java/com/linkedin/pinot/core/common/BlockValSet.java index 89aa7b2..6c6c8a7 100644 --- a/pinot-core/src/main/java/com/linkedin/pinot/core/common/BlockValSet.java +++ b/pinot-core/src/main/java/com/linkedin/pinot/core/common/BlockValSet.java @@ -30,7 +30,6 @@ public interface BlockValSet { /** * Get dictionary Ids for the given docIds. - * * @param inDocIds Input docIds * @param inStartPos Start index in inDocIds * @param inDocIdsSize Number of input doc ids @@ -41,68 +40,62 @@ public interface BlockValSet { /** * Get Integer values for the given docIds. - * * @param inDocIds Input docIds * @param inStartPos Start index in inDocIds * @param inDocIdsSize Number of input doc ids * @param outValues Output array * @param outStartPos Start position in outValues */ - void getIntValues(int[] inDocIds, int inStartPos, int inDocIdsSize, int[] outValues, int outStartPos); + long getIntValues(int[] inDocIds, int inStartPos, int inDocIdsSize, int[] outValues, int outStartPos); /** * Get long values for the given docIds. - * * @param inDocIds Input docIds * @param inStartPos Start index in inDocIds * @param inDocIdsSize Number of input doc ids * @param outValues Output array * @param outStartPos Start position in outValues */ - void getLongValues(int[] inDocIds, int inStartPos, int inDocIdsSize, long[] outValues, int outStartPos); + long getLongValues(int[] inDocIds, int inStartPos, int inDocIdsSize, long[] outValues, int outStartPos); /** * Get float values for the given docIds. - * * @param inDocIds Input docIds * @param inStartPos Start index in inDocIds * @param inDocIdsSize Number of input doc ids * @param outValues Output array * @param outStartPos Start position in outValues */ - void getFloatValues(int[] inDocIds, int inStartPos, int inDocIdsSize, float[] outValues, int outStartPos); + long getFloatValues(int[] inDocIds, int inStartPos, int inDocIdsSize, float[] outValues, int outStartPos); /** - * * @param inDocIds Input docIds * @param inStartPos Start index in inDocIds * @param inDocIdsSize Number of input doc ids * @param outValues Output array * @param outStartPos Start position in outValues */ - void getDoubleValues(int[] inDocIds, int inStartPos, int inDocIdsSize, double[] outValues, int outStartPos); + long getDoubleValues(int[] inDocIds, int inStartPos, int inDocIdsSize, double[] outValues, int outStartPos); /** * Get string values for the given docIds. - * * @param inDocIds Input docIds * @param inStartPos Start index in inDocIds * @param inDocIdsSize Number of input doc ids * @param outValues Output array * @param outStartPos Start position in outValues */ - void getStringValues(int[] inDocIds, int inStartPos, int inDocIdsSize, String[] outValues, int outStartPos); + long getStringValues(int[] inDocIds, int inStartPos, int inDocIdsSize, String[] outValues, int outStartPos); /** * Get byte[] values for the given docIds. - * * @param inDocIds Input docIds * @param inStartPos Start index in inDocIds * @param inDocIdsSize Number of input doc ids * @param outValues Output array * @param outStartPos Start position in outValues */ - void getBytesValues(int[] inDocIds, int inStartPos, int inDocIdsSize, byte[][] outValues, int outStartPos); + long getBytesValues(int[] inDocIds, int inStartPos, int inDocIdsSize, byte[][] outValues, int outStartPos); /** * SINGLE-VALUED COLUMN APIs diff --git a/pinot-core/src/main/java/com/linkedin/pinot/core/operator/docvalsets/SingleValueSet.java b/pinot-core/src/main/java/com/linkedin/pinot/core/operator/docvalsets/SingleValueSet.java index 0480454..0fd13d6 100644 --- a/pinot-core/src/main/java/com/linkedin/pinot/core/operator/docvalsets/SingleValueSet.java +++ b/pinot-core/src/main/java/com/linkedin/pinot/core/operator/docvalsets/SingleValueSet.java @@ -46,7 +46,7 @@ public final class SingleValueSet extends BaseBlockValSet { } @Override - public void getIntValues(int[] inDocIds, int inStartPos, int inDocIdsSize, int[] outValues, int outStartPos) { + public long getIntValues(int[] inDocIds, int inStartPos, int inDocIdsSize, int[] outValues, int outStartPos) { int inEndPos = inStartPos + inDocIdsSize; ReaderContext context = _reader.createContext(); if (_dataType == DataType.INT) { @@ -56,10 +56,11 @@ public final class SingleValueSet extends BaseBlockValSet { } else { throw new UnsupportedOperationException(); } + return inDocIdsSize * Integer.BYTES; } @Override - public void getLongValues(int[] inDocIds, int inStartPos, int inDocIdsSize, long[] outValues, int outStartPos) { + public long getLongValues(int[] inDocIds, int inStartPos, int inDocIdsSize, long[] outValues, int outStartPos) { int inEndPos = inStartPos + inDocIdsSize; ReaderContext context = _reader.createContext(); switch (_dataType) { @@ -67,19 +68,19 @@ public final class SingleValueSet extends BaseBlockValSet { for (int i = inStartPos; i < inEndPos; i++) { outValues[outStartPos++] = _reader.getInt(inDocIds[i], context); } - break; + return inDocIdsSize * Integer.BYTES; case LONG: for (int i = inStartPos; i < inEndPos; i++) { outValues[outStartPos++] = _reader.getLong(inDocIds[i], context); } - break; + return inDocIdsSize * Long.BYTES; default: throw new UnsupportedOperationException(); } } @Override - public void getFloatValues(int[] inDocIds, int inStartPos, int inDocIdsSize, float[] outValues, int outStartPos) { + public long getFloatValues(int[] inDocIds, int inStartPos, int inDocIdsSize, float[] outValues, int outStartPos) { int inEndPos = inStartPos + inDocIdsSize; ReaderContext context = _reader.createContext(); switch (_dataType) { @@ -87,24 +88,24 @@ public final class SingleValueSet extends BaseBlockValSet { for (int i = inStartPos; i < inEndPos; i++) { outValues[outStartPos++] = _reader.getInt(inDocIds[i], context); } - break; + return inDocIdsSize * Integer.BYTES; case LONG: for (int i = inStartPos; i < inEndPos; i++) { outValues[outStartPos++] = _reader.getLong(inDocIds[i], context); } - break; + return inDocIdsSize * Long.BYTES; case FLOAT: for (int i = inStartPos; i < inEndPos; i++) { outValues[outStartPos++] = _reader.getFloat(inDocIds[i], context); } - break; + return inDocIdsSize * Float.BYTES; default: throw new UnsupportedOperationException(); } } @Override - public void getDoubleValues(int[] inDocIds, int inStartPos, int inDocIdsSize, double[] outValues, int outStartPos) { + public long getDoubleValues(int[] inDocIds, int inStartPos, int inDocIdsSize, double[] outValues, int outStartPos) { int inEndPos = inStartPos + inDocIdsSize; ReaderContext context = _reader.createContext(); switch (_dataType) { @@ -112,48 +113,57 @@ public final class SingleValueSet extends BaseBlockValSet { for (int i = inStartPos; i < inEndPos; i++) { outValues[outStartPos++] = _reader.getInt(inDocIds[i], context); } - break; + return inDocIdsSize * Integer.BYTES; case LONG: for (int i = inStartPos; i < inEndPos; i++) { outValues[outStartPos++] = _reader.getLong(inDocIds[i], context); } - break; + return inDocIdsSize * Long.BYTES; case FLOAT: for (int i = inStartPos; i < inEndPos; i++) { outValues[outStartPos++] = _reader.getFloat(inDocIds[i], context); } - break; + return inDocIdsSize * Float.BYTES; case DOUBLE: for (int i = inStartPos; i < inEndPos; i++) { outValues[outStartPos++] = _reader.getDouble(inDocIds[i], context); } - break; + return inDocIdsSize * Double.BYTES; default: throw new UnsupportedOperationException(); } } @Override - public void getStringValues(int[] inDocIds, int inStartPos, int inDocIdsSize, String[] outValues, int outStartPos) { + public long getStringValues(int[] inDocIds, int inStartPos, int inDocIdsSize, String[] outValues, int outStartPos) { + + long bytesRead = 0; int inEndPos = inStartPos + inDocIdsSize; ReaderContext context = _reader.createContext(); if (_dataType == DataType.STRING) { for (int i = inStartPos; i < inEndPos; i++) { - outValues[outStartPos++] = _reader.getString(inDocIds[i], context); + String val = _reader.getString(inDocIds[i], context); + outValues[outStartPos++] = val; + bytesRead += val.length(); } + return bytesRead; } else { throw new UnsupportedOperationException(); } } @Override - public void getBytesValues(int[] inDocIds, int inStartPos, int inDocIdsSize, byte[][] outValues, int outStartPos) { + public long getBytesValues(int[] inDocIds, int inStartPos, int inDocIdsSize, byte[][] outValues, int outStartPos) { int inEndPos = inStartPos + inDocIdsSize; + long bytesRead = 0; ReaderContext context = _reader.createContext(); if (_dataType.equals(DataType.BYTES)) { for (int i = inStartPos; i < inEndPos; i++) { - outValues[outStartPos++] = _reader.getBytes(inDocIds[i], context); + byte[] val = _reader.getBytes(inDocIds[i], context); + outValues[outStartPos++] = val; + bytesRead += val.length; } + return bytesRead; } else { throw new UnsupportedOperationException(); } diff --git a/pinot-core/src/main/java/com/linkedin/pinot/core/segment/index/readers/BaseDictionary.java b/pinot-core/src/main/java/com/linkedin/pinot/core/segment/index/readers/BaseDictionary.java index 181168c..b919884 100644 --- a/pinot-core/src/main/java/com/linkedin/pinot/core/segment/index/readers/BaseDictionary.java +++ b/pinot-core/src/main/java/com/linkedin/pinot/core/segment/index/readers/BaseDictionary.java @@ -54,50 +54,62 @@ public abstract class BaseDictionary implements Dictionary { } @Override - public void readIntValues(int[] dictIds, int inStartPos, int length, int[] outValues, int outStartPos) { + public long readIntValues(int[] dictIds, int inStartPos, int length, int[] outValues, int outStartPos) { int inEndPos = inStartPos + length; for (int i = inStartPos; i < inEndPos; i++) { outValues[outStartPos++] = getIntValue(dictIds[i]); } + return length * Integer.BYTES; } @Override - public void readLongValues(int[] dictIds, int inStartPos, int length, long[] outValues, int outStartPos) { + public long readLongValues(int[] dictIds, int inStartPos, int length, long[] outValues, int outStartPos) { int inEndPos = inStartPos + length; for (int i = inStartPos; i < inEndPos; i++) { outValues[outStartPos++] = getLongValue(dictIds[i]); } + return length * Long.BYTES; } @Override - public void readFloatValues(int[] dictIds, int inStartPos, int length, float[] outValues, int outStartPos) { + public long readFloatValues(int[] dictIds, int inStartPos, int length, float[] outValues, int outStartPos) { int inEndPos = inStartPos + length; for (int i = inStartPos; i < inEndPos; i++) { outValues[outStartPos++] = getFloatValue(dictIds[i]); } + return length * Float.BYTES; } @Override - public void readDoubleValues(int[] dictIds, int inStartPos, int length, double[] outValues, int outStartPos) { + public long readDoubleValues(int[] dictIds, int inStartPos, int length, double[] outValues, int outStartPos) { int inEndPos = inStartPos + length; for (int i = inStartPos; i < inEndPos; i++) { outValues[outStartPos++] = getDoubleValue(dictIds[i]); } + return length * Double.BYTES; } @Override - public void readStringValues(int[] dictIds, int inStartPos, int length, String[] outValues, int outStartPos) { + public long readStringValues(int[] dictIds, int inStartPos, int length, String[] outValues, int outStartPos) { + long bytesRead = 0; int inEndPos = inStartPos + length; for (int i = inStartPos; i < inEndPos; i++) { - outValues[outStartPos++] = getStringValue(dictIds[i]); + String str = getStringValue(dictIds[i]); + outValues[outStartPos++] = str; + bytesRead += str.length(); } + return bytesRead; } @Override - public void readBytesValues(int[] dictIds, int inStartPos, int length, byte[][] outValues, int outStartPos) { + public long readBytesValues(int[] dictIds, int inStartPos, int length, byte[][] outValues, int outStartPos) { + long bytesRead = 0; int inEndPos = inStartPos + length; for (int i = inStartPos; i < inEndPos; i++) { - outValues[outStartPos++] = getBytesValue(dictIds[i]); + byte[] val = getBytesValue(dictIds[i]); + outValues[outStartPos++] = val; + bytesRead += val.length; } + return bytesRead; } } diff --git a/pinot-core/src/main/java/com/linkedin/pinot/core/segment/index/readers/BytesDictionary.java b/pinot-core/src/main/java/com/linkedin/pinot/core/segment/index/readers/BytesDictionary.java index b5cc02c..bce06cd 100644 --- a/pinot-core/src/main/java/com/linkedin/pinot/core/segment/index/readers/BytesDictionary.java +++ b/pinot-core/src/main/java/com/linkedin/pinot/core/segment/index/readers/BytesDictionary.java @@ -49,10 +49,14 @@ public class BytesDictionary extends ImmutableDictionaryReader { } @Override - public void readBytesValues(int[] dictIds, int inStartPos, int length, byte[][] outValues, int outStartPos) { + public long readBytesValues(int[] dictIds, int inStartPos, int length, byte[][] outValues, int outStartPos) { + long bytesRead = 0; int inEndPos = inStartPos + length; for (int i = inStartPos; i < inEndPos; i++) { - outValues[outStartPos++] = getBytes(dictIds[i], getBuffer()); + byte[] val = getBytes(dictIds[i], getBuffer()); + outValues[outStartPos++] = val; + bytesRead += val.length; } + return bytesRead; } } diff --git a/pinot-core/src/main/java/com/linkedin/pinot/core/segment/index/readers/Dictionary.java b/pinot-core/src/main/java/com/linkedin/pinot/core/segment/index/readers/Dictionary.java index 80145fb..257749f 100644 --- a/pinot-core/src/main/java/com/linkedin/pinot/core/segment/index/readers/Dictionary.java +++ b/pinot-core/src/main/java/com/linkedin/pinot/core/segment/index/readers/Dictionary.java @@ -50,16 +50,16 @@ public interface Dictionary extends Closeable { // Batch read APIs - void readIntValues(int[] dictIds, int inStartPos, int length, int[] outValues, int outStartPos); + long readIntValues(int[] dictIds, int inStartPos, int length, int[] outValues, int outStartPos); - void readLongValues(int[] dictIds, int inStartPos, int length, long[] outValues, int outStartPos); + long readLongValues(int[] dictIds, int inStartPos, int length, long[] outValues, int outStartPos); - void readFloatValues(int[] dictIds, int inStartPos, int length, float[] outValues, int outStartPos); + long readFloatValues(int[] dictIds, int inStartPos, int length, float[] outValues, int outStartPos); - void readDoubleValues(int[] dictIds, int inStartPos, int length, double[] outValues, int outStartPos); + long readDoubleValues(int[] dictIds, int inStartPos, int length, double[] outValues, int outStartPos); - void readStringValues(int[] dictIds, int inStartPos, int length, String[] outValues, int outStartPos); + long readStringValues(int[] dictIds, int inStartPos, int length, String[] outValues, int outStartPos); - void readBytesValues(int[] dictIds, int inStartPos, int length, byte[][] outValues, int outStartPos); + long readBytesValues(int[] dictIds, int inStartPos, int length, byte[][] outValues, int outStartPos); } diff --git a/pinot-core/src/main/java/com/linkedin/pinot/core/segment/index/readers/StringDictionary.java b/pinot-core/src/main/java/com/linkedin/pinot/core/segment/index/readers/StringDictionary.java index 2460547..7a7dab1 100644 --- a/pinot-core/src/main/java/com/linkedin/pinot/core/segment/index/readers/StringDictionary.java +++ b/pinot-core/src/main/java/com/linkedin/pinot/core/segment/index/readers/StringDictionary.java @@ -51,11 +51,15 @@ public class StringDictionary extends ImmutableDictionaryReader { } @Override - public void readStringValues(int[] dictIds, int inStartPos, int length, String[] outValues, int outStartPos) { + public long readStringValues(int[] dictIds, int inStartPos, int length, String[] outValues, int outStartPos) { + long bytesRead = 0; byte[] buffer = getBuffer(); int inEndPos = inStartPos + length; for (int i = inStartPos; i < inEndPos; i++) { - outValues[outStartPos++] = getUnpaddedString(dictIds[i], buffer); + String val = getUnpaddedString(dictIds[i], buffer); + outValues[outStartPos++] = val; + bytesRead += val.length(); } + return bytesRead; } } --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
