mikemccand commented on a change in pull request #973: LUCENE-9027: Use SIMD 
instructions to decode postings.
URL: https://github.com/apache/lucene-solr/pull/973#discussion_r345752696
 
 

 ##########
 File path: 
lucene/core/src/java/org/apache/lucene/codecs/lucene84/gen_ForUtil.py
 ##########
 @@ -0,0 +1,341 @@
+#! /usr/bin/env python
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from fractions import gcd
+
+"""Code generation for ForUtil.java"""
+
+MAX_SPECIALIZED_BITS_PER_VALUE = 24
+OUTPUT_FILE = "ForUtil.java"
+HEADER = """// This file has been automatically generated, DO NOT EDIT
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.codecs.lucene84;
+
+import java.io.IOException;
+import java.nio.ByteOrder;
+import java.util.Arrays;
+import java.util.function.IntToLongFunction;
+
+import org.apache.lucene.store.DataInput;
+import org.apache.lucene.store.DataOutput;
+
+// Inspired from https://fulmicoton.com/posts/bitpacking/
+// Encodes multiple integers in a long to get SIMD-like speedups.
+// If bitsPerValue <= 8 then we pack 8 ints per long
+// else if bitsPerValue <= 16 we pack 4 ints per long
+// else we pack 2 ints per long
+final class ForUtil {
+
+  static final int BLOCK_SIZE = 128;
+  private static final int BLOCK_SIZE_LOG2 = 7;
+
+  private static long expandMask32(long mask32) {
+    return mask32 | (mask32 << 32);
+  }
+
+  private static long expandMask16(long mask16) {
+    return expandMask32(mask16 | (mask16 << 16));
+  }
+
+  private static long expandMask8(long mask8) {
+    return expandMask16(mask8 | (mask8 << 8));
+  }
+
+  private static long mask32(int bitsPerValue) {
+    return expandMask32((1L << bitsPerValue) - 1);
+  }
+
+  private static long mask16(int bitsPerValue) {
+    return expandMask16((1L << bitsPerValue) - 1);
+  }
+
+  private static long mask8(int bitsPerValue) {
+    return expandMask8((1L << bitsPerValue) - 1);
+  }
+
+  private static void expand8(long[] arr) {
+    for (int i = 0; i < 16; ++i) {
+      long l = arr[i];
+      arr[i] = (l >>> 56) & 0xFFL;
+      arr[16+i] = (l >>> 48) & 0xFFL;
+      arr[32+i] = (l >>> 40) & 0xFFL;
+      arr[48+i] = (l >>> 32) & 0xFFL;
+      arr[64+i] = (l >>> 24) & 0xFFL;
+      arr[80+i] = (l >>> 16) & 0xFFL;
+      arr[96+i] = (l >>> 8) & 0xFFL;
+      arr[112+i] = l & 0xFFL;
+    }
+  }
+
+  private static void collapse8(long[] arr) {
+    for (int i = 0; i < 16; ++i) {
+      arr[i] = (arr[i] << 56) | (arr[16+i] << 48) | (arr[32+i] << 40) | 
(arr[48+i] << 32) | (arr[64+i] << 24) | (arr[80+i] << 16) | (arr[96+i] << 8) | 
arr[112+i];
+    }
+  }
+
+  private static void expand16(long[] arr) {
+    for (int i = 0; i < 32; ++i) {
+      long l = arr[i];
+      arr[i] = (l >>> 48) & 0xFFFFL;
+      arr[32+i] = (l >>> 32) & 0xFFFFL;
+      arr[64+i] = (l >>> 16) & 0xFFFFL;
+      arr[96+i] = l & 0xFFFFL;
+    }
+  }
+
+  private static void collapse16(long[] arr) {
+    for (int i = 0; i < 32; ++i) {
+      arr[i] = (arr[i] << 48) | (arr[32+i] << 32) | (arr[64+i] << 16) | 
arr[96+i];
+    }
+  }
+
+  private static void expand32(long[] arr) {
+    for (int i = 0; i < 64; ++i) {
+      long l = arr[i];
+      arr[i] = l >>> 32;
+      arr[64 + i] = l & 0xFFFFFFFFL;
+    }
+  }
+
+  private static void collapse32(long[] arr) {
+    for (int i = 0; i < 64; ++i) {
+      arr[i] = (arr[i] << 32) | arr[64+i];
+    }
+  }
+
+  private final ByteOrder byteOrder;
+  private final long[] tmp = new long[BLOCK_SIZE/2];
+
+  ForUtil(ByteOrder byteOrder) {
+    this.byteOrder = byteOrder;
+  }
+
+  /**
+   * Encode 128 8-bits integers from {@code data} into {@code out}.
+   */
+  void encode(long[] longs, int bitsPerValue, DataOutput out) throws 
IOException {
+    final int nextPrimitive;
+    final int numLongs;
+    final IntToLongFunction maskFunction;
+    if (bitsPerValue <= 8) {
+      nextPrimitive = 8;
+      numLongs = BLOCK_SIZE / 8;
+      maskFunction = ForUtil::mask8;
+      collapse8(longs);
+    } else if (bitsPerValue <= 16) {
+      nextPrimitive = 16;
+      numLongs = BLOCK_SIZE / 4;
+      maskFunction = ForUtil::mask16;
+      collapse16(longs);
+    } else {
+      nextPrimitive = 32;
+      numLongs = BLOCK_SIZE / 2;
+      maskFunction = ForUtil::mask32;
+      collapse32(longs);
+    }
+
+    final int numLongsPerShift = bitsPerValue * 2;
+    Arrays.fill(tmp, 0L);
+    int idx = 0;
+    for (int shift = nextPrimitive - bitsPerValue; shift >= 0; shift -= 
bitsPerValue) {
+      for (int i = 0; i < numLongsPerShift; ++i) {
+        tmp[i] |= longs[idx++] << shift;
+      }
+    }
+
+    final int remainingBitsPerLong = nextPrimitive % bitsPerValue;
+    int tmpIdx = 0;
+    int remainingBitsPerValue = bitsPerValue;
+    while (idx < numLongs) {
+      if (remainingBitsPerValue > remainingBitsPerLong) {
+        remainingBitsPerValue -= remainingBitsPerLong;
+        tmp[tmpIdx++] |= (longs[idx] >>> remainingBitsPerValue) & 
maskFunction.applyAsLong(remainingBitsPerLong);
+        if (remainingBitsPerValue == 0) {
+          idx++;
+          remainingBitsPerValue = bitsPerValue;
+        }
+      } else {
+        tmp[tmpIdx] |= (longs[idx++] & 
maskFunction.applyAsLong(remainingBitsPerValue)) << (remainingBitsPerLong - 
remainingBitsPerValue);
+        remainingBitsPerValue = bitsPerValue - remainingBitsPerLong + 
remainingBitsPerValue;
+        tmp[tmpIdx++] |= (longs[idx] >>> remainingBitsPerValue) & 
maskFunction.applyAsLong(bitsPerValue - remainingBitsPerValue);
+      }
+    }
+
+    for (int i = 0; i < numLongsPerShift; ++i) {
+      long l = tmp[i];
+      if (byteOrder != ByteOrder.BIG_ENDIAN) {
 
 Review comment:
   This is commonly the case -- x86 is little-endian.  I suppose we could make 
things a bit faster by reversing this, i.e. fixing our Python code generator to 
write bytes in little-endian, and then reverse only if the current server is 
big-endian (unusual).  But that's likely a minor gain...

----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


With regards,
Apache Git Services

---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscr...@lucene.apache.org
For additional commands, e-mail: issues-h...@lucene.apache.org

Reply via email to