juanka588 commented on a change in pull request #1282: Lucene 9236
URL: https://github.com/apache/lucene-solr/pull/1282#discussion_r383184671
##########
File path:
lucene/core/src/java/org/apache/lucene/codecs/lucene80/Lucene80DocValuesProducer.java
##########
@@ -121,1445 +116,80 @@ private void readFields(ChecksumIndexInput meta,
FieldInfos infos) throws IOExce
}
byte type = meta.readByte();
if (type == Lucene80DocValuesFormat.NUMERIC) {
- numerics.put(info.name, readNumeric(meta));
+ numerics.put(info.name, Lucene80NumericProducer.readNumeric(meta));
} else if (type == Lucene80DocValuesFormat.BINARY) {
- binaries.put(info.name, readBinary(meta));
+ binaries.put(info.name, Lucene80BinaryProducer.readBinary(meta,
version));
} else if (type == Lucene80DocValuesFormat.SORTED) {
- sorted.put(info.name, readSorted(meta));
+ sorted.put(info.name, Lucene80SortedSetProducer.readSorted(meta));
} else if (type == Lucene80DocValuesFormat.SORTED_SET) {
- sortedSets.put(info.name, readSortedSet(meta));
+ sortedSets.put(info.name,
Lucene80SortedSetProducer.readSortedSet(meta));
} else if (type == Lucene80DocValuesFormat.SORTED_NUMERIC) {
- sortedNumerics.put(info.name, readSortedNumeric(meta));
+ sortedNumerics.put(info.name,
Lucene80NumericProducer.readSortedNumeric(meta));
} else {
throw new CorruptIndexException("invalid type: " + type, meta);
}
}
}
- private NumericEntry readNumeric(ChecksumIndexInput meta) throws IOException
{
- NumericEntry entry = new NumericEntry();
- readNumeric(meta, entry);
- return entry;
- }
-
- private void readNumeric(ChecksumIndexInput meta, NumericEntry entry) throws
IOException {
- entry.docsWithFieldOffset = meta.readLong();
- entry.docsWithFieldLength = meta.readLong();
- entry.jumpTableEntryCount = meta.readShort();
- entry.denseRankPower = meta.readByte();
- entry.numValues = meta.readLong();
- int tableSize = meta.readInt();
- if (tableSize > 256) {
- throw new CorruptIndexException("invalid table size: " + tableSize,
meta);
- }
- if (tableSize >= 0) {
- entry.table = new long[tableSize];
- ramBytesUsed += RamUsageEstimator.sizeOf(entry.table);
- for (int i = 0; i < tableSize; ++i) {
- entry.table[i] = meta.readLong();
- }
- }
- if (tableSize < -1) {
- entry.blockShift = -2 - tableSize;
- } else {
- entry.blockShift = -1;
- }
- entry.bitsPerValue = meta.readByte();
- entry.minValue = meta.readLong();
- entry.gcd = meta.readLong();
- entry.valuesOffset = meta.readLong();
- entry.valuesLength = meta.readLong();
- entry.valueJumpTableOffset = meta.readLong();
- }
-
- private BinaryEntry readBinary(ChecksumIndexInput meta) throws IOException {
- BinaryEntry entry = new BinaryEntry();
- entry.dataOffset = meta.readLong();
- entry.dataLength = meta.readLong();
- entry.docsWithFieldOffset = meta.readLong();
- entry.docsWithFieldLength = meta.readLong();
- entry.jumpTableEntryCount = meta.readShort();
- entry.denseRankPower = meta.readByte();
- entry.numDocsWithField = meta.readInt();
- entry.minLength = meta.readInt();
- entry.maxLength = meta.readInt();
- if ((version >= Lucene80DocValuesFormat.VERSION_BIN_COMPRESSED &&
entry.numDocsWithField > 0) || entry.minLength < entry.maxLength) {
- entry.addressesOffset = meta.readLong();
-
- // Old count of uncompressed addresses
- long numAddresses = entry.numDocsWithField + 1L;
- // New count of compressed addresses - the number of compresseed blocks
- if (version >= Lucene80DocValuesFormat.VERSION_BIN_COMPRESSED) {
- entry.numCompressedChunks = meta.readVInt();
- entry.docsPerChunkShift = meta.readVInt();
- entry.maxUncompressedChunkSize = meta.readVInt();
- numAddresses = entry.numCompressedChunks;
- }
-
- final int blockShift = meta.readVInt();
- entry.addressesMeta = DirectMonotonicReader.loadMeta(meta, numAddresses,
blockShift);
- ramBytesUsed += entry.addressesMeta.ramBytesUsed();
- entry.addressesLength = meta.readLong();
- }
- return entry;
- }
-
- private SortedEntry readSorted(ChecksumIndexInput meta) throws IOException {
- SortedEntry entry = new SortedEntry();
- entry.docsWithFieldOffset = meta.readLong();
- entry.docsWithFieldLength = meta.readLong();
- entry.jumpTableEntryCount = meta.readShort();
- entry.denseRankPower = meta.readByte();
- entry.numDocsWithField = meta.readInt();
- entry.bitsPerValue = meta.readByte();
- entry.ordsOffset = meta.readLong();
- entry.ordsLength = meta.readLong();
- readTermDict(meta, entry);
- return entry;
- }
-
- private SortedSetEntry readSortedSet(ChecksumIndexInput meta) throws
IOException {
- SortedSetEntry entry = new SortedSetEntry();
- byte multiValued = meta.readByte();
- switch (multiValued) {
- case 0: // singlevalued
- entry.singleValueEntry = readSorted(meta);
- return entry;
- case 1: // multivalued
- break;
- default:
- throw new CorruptIndexException("Invalid multiValued flag: " +
multiValued, meta);
- }
- entry.docsWithFieldOffset = meta.readLong();
- entry.docsWithFieldLength = meta.readLong();
- entry.jumpTableEntryCount = meta.readShort();
- entry.denseRankPower = meta.readByte();
- entry.bitsPerValue = meta.readByte();
- entry.ordsOffset = meta.readLong();
- entry.ordsLength = meta.readLong();
- entry.numDocsWithField = meta.readInt();
- entry.addressesOffset = meta.readLong();
- final int blockShift = meta.readVInt();
- entry.addressesMeta = DirectMonotonicReader.loadMeta(meta,
entry.numDocsWithField + 1, blockShift);
- ramBytesUsed += entry.addressesMeta.ramBytesUsed();
- entry.addressesLength = meta.readLong();
- readTermDict(meta, entry);
- return entry;
- }
-
- private static void readTermDict(ChecksumIndexInput meta, TermsDictEntry
entry) throws IOException {
- entry.termsDictSize = meta.readVLong();
- entry.termsDictBlockShift = meta.readInt();
- final int blockShift = meta.readInt();
- final long addressesSize = (entry.termsDictSize + (1L <<
entry.termsDictBlockShift) - 1) >>> entry.termsDictBlockShift;
- entry.termsAddressesMeta = DirectMonotonicReader.loadMeta(meta,
addressesSize, blockShift);
- entry.maxTermLength = meta.readInt();
- entry.termsDataOffset = meta.readLong();
- entry.termsDataLength = meta.readLong();
- entry.termsAddressesOffset = meta.readLong();
- entry.termsAddressesLength = meta.readLong();
- entry.termsDictIndexShift = meta.readInt();
- final long indexSize = (entry.termsDictSize + (1L <<
entry.termsDictIndexShift) - 1) >>> entry.termsDictIndexShift;
- entry.termsIndexAddressesMeta = DirectMonotonicReader.loadMeta(meta, 1 +
indexSize, blockShift);
- entry.termsIndexOffset = meta.readLong();
- entry.termsIndexLength = meta.readLong();
- entry.termsIndexAddressesOffset = meta.readLong();
- entry.termsIndexAddressesLength = meta.readLong();
- }
-
- private SortedNumericEntry readSortedNumeric(ChecksumIndexInput meta) throws
IOException {
- SortedNumericEntry entry = new SortedNumericEntry();
- readNumeric(meta, entry);
- entry.numDocsWithField = meta.readInt();
- if (entry.numDocsWithField != entry.numValues) {
- entry.addressesOffset = meta.readLong();
- final int blockShift = meta.readVInt();
- entry.addressesMeta = DirectMonotonicReader.loadMeta(meta,
entry.numDocsWithField + 1, blockShift);
- ramBytesUsed += entry.addressesMeta.ramBytesUsed();
- entry.addressesLength = meta.readLong();
- }
- return entry;
- }
-
@Override
public void close() throws IOException {
data.close();
}
- private static class NumericEntry {
- long[] table;
- int blockShift;
- byte bitsPerValue;
- long docsWithFieldOffset;
- long docsWithFieldLength;
- short jumpTableEntryCount;
- byte denseRankPower;
- long numValues;
- long minValue;
- long gcd;
- long valuesOffset;
- long valuesLength;
- long valueJumpTableOffset; // -1 if no jump-table
- }
-
- private static class BinaryEntry {
- long dataOffset;
- long dataLength;
- long docsWithFieldOffset;
- long docsWithFieldLength;
- short jumpTableEntryCount;
- byte denseRankPower;
- int numDocsWithField;
- int minLength;
- int maxLength;
- long addressesOffset;
- long addressesLength;
- DirectMonotonicReader.Meta addressesMeta;
- int numCompressedChunks;
- int docsPerChunkShift;
- int maxUncompressedChunkSize;
- }
-
- private static class TermsDictEntry {
- long termsDictSize;
- int termsDictBlockShift;
- DirectMonotonicReader.Meta termsAddressesMeta;
- int maxTermLength;
- long termsDataOffset;
- long termsDataLength;
- long termsAddressesOffset;
- long termsAddressesLength;
- int termsDictIndexShift;
- DirectMonotonicReader.Meta termsIndexAddressesMeta;
- long termsIndexOffset;
- long termsIndexLength;
- long termsIndexAddressesOffset;
- long termsIndexAddressesLength;
- }
-
- private static class SortedEntry extends TermsDictEntry {
- long docsWithFieldOffset;
- long docsWithFieldLength;
- short jumpTableEntryCount;
- byte denseRankPower;
- int numDocsWithField;
- byte bitsPerValue;
- long ordsOffset;
- long ordsLength;
- }
-
- private static class SortedSetEntry extends TermsDictEntry {
- SortedEntry singleValueEntry;
- long docsWithFieldOffset;
- long docsWithFieldLength;
- short jumpTableEntryCount;
- byte denseRankPower;
- int numDocsWithField;
- byte bitsPerValue;
- long ordsOffset;
- long ordsLength;
- DirectMonotonicReader.Meta addressesMeta;
- long addressesOffset;
- long addressesLength;
- }
-
- private static class SortedNumericEntry extends NumericEntry {
- int numDocsWithField;
- DirectMonotonicReader.Meta addressesMeta;
- long addressesOffset;
- long addressesLength;
- }
-
@Override
public long ramBytesUsed() {
- return ramBytesUsed;
- }
-
- @Override
- public NumericDocValues getNumeric(FieldInfo field) throws IOException {
- NumericEntry entry = numerics.get(field.name);
- return getNumeric(entry);
- }
-
- private static abstract class DenseNumericDocValues extends NumericDocValues
{
-
- final int maxDoc;
- int doc = -1;
-
- DenseNumericDocValues(int maxDoc) {
- this.maxDoc = maxDoc;
- }
-
- @Override
- public int docID() {
- return doc;
- }
-
- @Override
- public int nextDoc() throws IOException {
- return advance(doc + 1);
- }
-
- @Override
- public int advance(int target) throws IOException {
- if (target >= maxDoc) {
- return doc = NO_MORE_DOCS;
- }
- return doc = target;
- }
-
- @Override
- public boolean advanceExact(int target) {
- doc = target;
- return true;
- }
-
- @Override
- public long cost() {
- return maxDoc;
- }
-
- }
-
- private static abstract class SparseNumericDocValues extends
NumericDocValues {
-
- final IndexedDISI disi;
-
- SparseNumericDocValues(IndexedDISI disi) {
- this.disi = disi;
+ long ramBytesUsed = BASE_RAM_USAGE;
+ for (Accountable accountable : numerics.values()) {
Review comment:
now each entry implements accountable
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
