Adding a fix/hack to be able to externalize the FST data arrays in Lucene.
Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/393c9f53 Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/393c9f53 Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/393c9f53 Branch: refs/heads/apache-blur-0.2 Commit: 393c9f53d2b6e0e004a2813e3cc7ad94fd4f01bc Parents: d0e26ca Author: Aaron McCurry <[email protected]> Authored: Thu Feb 20 21:39:49 2014 -0500 Committer: Aaron McCurry <[email protected]> Committed: Thu Feb 20 21:39:49 2014 -0500 ---------------------------------------------------------------------- .../org/apache/blur/lucene/fst/ByteArray.java | 42 ++ .../blur/lucene/fst/ByteArrayFactory.java | 70 +++ .../blur/lucene/fst/ByteArrayPrimitive.java | 72 +++ .../lucene/fst/ByteArrayPrimitiveFactory.java | 32 ++ .../org/apache/lucene/util/fst/BytesStore.java | 561 +++++++++++++++++++ .../org/apache/blur/utils/BlurConstants.java | 1 + .../src/main/resources/blur-default.properties | 3 + 7 files changed, 781 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/393c9f53/blur-store/src/main/java/org/apache/blur/lucene/fst/ByteArray.java ---------------------------------------------------------------------- diff --git a/blur-store/src/main/java/org/apache/blur/lucene/fst/ByteArray.java b/blur-store/src/main/java/org/apache/blur/lucene/fst/ByteArray.java new file mode 100644 index 0000000..5d28e8c --- /dev/null +++ b/blur-store/src/main/java/org/apache/blur/lucene/fst/ByteArray.java @@ -0,0 +1,42 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.blur.lucene.fst; + +import java.io.IOException; + +import org.apache.lucene.store.DataInput; +import org.apache.lucene.store.DataOutput; + +public abstract class ByteArray { + + public abstract int length(); + + public abstract void put(int position, byte b); + + public abstract byte get(int position); + + public abstract void put(int position, byte[] b, int offset, int len); + + public abstract void get(int position, byte[] b, int offset, int len); + + public abstract void readBytes(DataInput in, int offset, int length) throws IOException; + + public abstract void writeBytes(DataOutput out, int offset, int length) throws IOException; + + public abstract void copy(int position, ByteArray dest, int destOffset, int len); + +} http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/393c9f53/blur-store/src/main/java/org/apache/blur/lucene/fst/ByteArrayFactory.java ---------------------------------------------------------------------- diff --git a/blur-store/src/main/java/org/apache/blur/lucene/fst/ByteArrayFactory.java b/blur-store/src/main/java/org/apache/blur/lucene/fst/ByteArrayFactory.java new file mode 100644 index 0000000..9bdb745 --- /dev/null +++ b/blur-store/src/main/java/org/apache/blur/lucene/fst/ByteArrayFactory.java @@ -0,0 +1,70 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.blur.lucene.fst; + +import static org.apache.blur.utils.BlurConstants.BLUR_LUCENE_FST_BYTEARRAY_FACTORY; + +import java.io.IOException; +import java.lang.reflect.Constructor; + +import org.apache.blur.BlurConfiguration; +import org.apache.blur.log.Log; +import org.apache.blur.log.LogFactory; + +public abstract class ByteArrayFactory { + + private static Log LOG = LogFactory.getLog(ByteArrayFactory.class); + + private final static ByteArrayFactory _factory; + + static { + BlurConfiguration configuration; + try { + configuration = new BlurConfiguration(); + } catch (IOException e) { + LOG.error("Error while trying to open blur config.", e); + try { + configuration = new BlurConfiguration(false); + } catch (IOException ex) { + throw new RuntimeException(); + } + } + ByteArrayFactory factory = new ByteArrayPrimitiveFactory(configuration); + try { + String className = configuration.get(BLUR_LUCENE_FST_BYTEARRAY_FACTORY); + if (!(className == null || className.isEmpty())) { + Class<?> clazz = Class.forName(className); + Constructor<?> constructor = clazz.getConstructor(new Class[] { BlurConfiguration.class }); + factory = (ByteArrayFactory) constructor.newInstance(new Object[] { constructor }); + } + } catch (Exception e) { + LOG.error("Error while trying create new bytearray factory for lucene bytestore.", e); + } + _factory = factory; + } + + public ByteArrayFactory(BlurConfiguration configuration) { + + } + + public abstract ByteArray newByteArray(int size); + + public static ByteArrayFactory getDefaultFactory() { + return _factory; + } + +} http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/393c9f53/blur-store/src/main/java/org/apache/blur/lucene/fst/ByteArrayPrimitive.java ---------------------------------------------------------------------- diff --git a/blur-store/src/main/java/org/apache/blur/lucene/fst/ByteArrayPrimitive.java b/blur-store/src/main/java/org/apache/blur/lucene/fst/ByteArrayPrimitive.java new file mode 100644 index 0000000..3c03879 --- /dev/null +++ b/blur-store/src/main/java/org/apache/blur/lucene/fst/ByteArrayPrimitive.java @@ -0,0 +1,72 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.blur.lucene.fst; + +import java.io.IOException; + +import org.apache.lucene.store.DataInput; +import org.apache.lucene.store.DataOutput; + +public class ByteArrayPrimitive extends ByteArray { + + private final byte[] _bytes; + + public ByteArrayPrimitive(int size) { + _bytes = new byte[size]; + } + + @Override + public int length() { + return _bytes.length; + } + + @Override + public void put(int position, byte b) { + _bytes[position] = b; + } + + @Override + public byte get(int position) { + return _bytes[position]; + } + + @Override + public void put(int position, byte[] b, int offset, int len) { + System.arraycopy(b, offset, _bytes, position, len); + } + + @Override + public void get(int position, byte[] b, int offset, int len) { + System.arraycopy(_bytes, position, b, offset, len); + } + + @Override + public void readBytes(DataInput in, int offset, int length) throws IOException { + in.readBytes(_bytes, offset, length); + } + + @Override + public void writeBytes(DataOutput out, int offset, int length) throws IOException { + out.writeBytes(_bytes, offset, length); + } + + @Override + public void copy(int position, ByteArray dest, int destOffset, int len) { + System.arraycopy(_bytes, position, ((ByteArrayPrimitive) dest)._bytes, destOffset, len); + } + +} http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/393c9f53/blur-store/src/main/java/org/apache/blur/lucene/fst/ByteArrayPrimitiveFactory.java ---------------------------------------------------------------------- diff --git a/blur-store/src/main/java/org/apache/blur/lucene/fst/ByteArrayPrimitiveFactory.java b/blur-store/src/main/java/org/apache/blur/lucene/fst/ByteArrayPrimitiveFactory.java new file mode 100644 index 0000000..e5f73a2 --- /dev/null +++ b/blur-store/src/main/java/org/apache/blur/lucene/fst/ByteArrayPrimitiveFactory.java @@ -0,0 +1,32 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.blur.lucene.fst; + +import org.apache.blur.BlurConfiguration; + +public class ByteArrayPrimitiveFactory extends ByteArrayFactory { + + public ByteArrayPrimitiveFactory(BlurConfiguration configuration) { + super(configuration); + } + + @Override + public ByteArray newByteArray(int size) { + return new ByteArrayPrimitive(size); + } + +} http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/393c9f53/blur-store/src/main/java/org/apache/lucene/util/fst/BytesStore.java ---------------------------------------------------------------------- diff --git a/blur-store/src/main/java/org/apache/lucene/util/fst/BytesStore.java b/blur-store/src/main/java/org/apache/lucene/util/fst/BytesStore.java new file mode 100644 index 0000000..6bac91b --- /dev/null +++ b/blur-store/src/main/java/org/apache/lucene/util/fst/BytesStore.java @@ -0,0 +1,561 @@ +package org.apache.lucene.util.fst; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import org.apache.blur.lucene.fst.ByteArray; +import org.apache.blur.lucene.fst.ByteArrayFactory; +import org.apache.lucene.store.DataInput; +import org.apache.lucene.store.DataOutput; + +// TODO: merge with PagedBytes, except PagedBytes doesn't +// let you read while writing which FST needs + +class BytesStore extends DataOutput { + + private final List<ByteArray> blocks = new ArrayList<ByteArray>(); + private final static ByteArrayFactory factory = ByteArrayFactory.getDefaultFactory(); + + private final int blockSize; + private final int blockBits; + private final int blockMask; + + private ByteArray current; + private int nextWrite; + + public BytesStore(int blockBits) { + this.blockBits = blockBits; + blockSize = 1 << blockBits; + blockMask = blockSize - 1; + nextWrite = blockSize; + } + + /** Pulls bytes from the provided IndexInput. */ + public BytesStore(DataInput in, long numBytes, int maxBlockSize) throws IOException { + int blockSize = 2; + int blockBits = 1; + while (blockSize < numBytes && blockSize < maxBlockSize) { + blockSize *= 2; + blockBits++; + } + this.blockBits = blockBits; + this.blockSize = blockSize; + this.blockMask = blockSize - 1; + long left = numBytes; + while (left > 0) { + final int chunk = (int) Math.min(blockSize, left); + ByteArray block = factory.newByteArray(chunk); + block.readBytes(in, 0, block.length()); + blocks.add(block); + left -= chunk; + } + + // So .getPosition still works + nextWrite = blocks.get(blocks.size() - 1).length(); + } + + /** + * Absolute write byte; you must ensure dest is < max position written so far. + */ + public void writeByte(int dest, byte b) { + int blockIndex = dest >> blockBits; + ByteArray block = blocks.get(blockIndex); + block.put(dest & blockMask, b); + } + + @Override + public void writeByte(byte b) { + if (nextWrite == blockSize) { + current = factory.newByteArray(blockSize); + blocks.add(current); + nextWrite = 0; + } + current.put(nextWrite++, b); + } + + @Override + public void writeBytes(byte[] b, int offset, int len) { + while (len > 0) { + int chunk = blockSize - nextWrite; + if (len <= chunk) { + arraycopy(b, offset, current, nextWrite, len); + nextWrite += len; + break; + } else { + if (chunk > 0) { + arraycopy(b, offset, current, nextWrite, chunk); + offset += chunk; + len -= chunk; + } + current = factory.newByteArray(blockSize); + blocks.add(current); + nextWrite = 0; + } + } + } + + int getBlockBits() { + return blockBits; + } + + void writeBytes(long dest, byte[] b, int offset, int len) { + ByteArray byteArray = factory.newByteArray(len); + byteArray.put(0, b, offset, len); + writeBytes(dest, byteArray, 0, len); + } + + /** + * Absolute writeBytes without changing the current position. Note: this + * cannot "grow" the bytes, so you must only call it on already written parts. + */ + void writeBytes(long dest, ByteArray b, int offset, int len) { + // System.out.println(" BS.writeBytes dest=" + dest + " offset=" + offset + + // " len=" + len); + assert dest + len <= getPosition() : "dest=" + dest + " pos=" + getPosition() + " len=" + len; + + // Note: weird: must go "backwards" because copyBytes + // calls us with overlapping src/dest. If we + // go forwards then we overwrite bytes before we can + // copy them: + + /* + * int blockIndex = dest >> blockBits; int upto = dest & blockMask; byte[] + * block = blocks.get(blockIndex); while (len > 0) { int chunk = blockSize - + * upto; System.out.println(" cycle chunk=" + chunk + " len=" + len); if + * (len <= chunk) { System.arraycopy(b, offset, block, upto, len); break; } + * else { System.arraycopy(b, offset, block, upto, chunk); offset += chunk; + * len -= chunk; blockIndex++; block = blocks.get(blockIndex); upto = 0; } } + */ + + final long end = dest + len; + int blockIndex = (int) (end >> blockBits); + int downTo = (int) (end & blockMask); + if (downTo == 0) { + blockIndex--; + downTo = blockSize; + } + ByteArray block = blocks.get(blockIndex); + + while (len > 0) { + // System.out.println(" cycle downTo=" + downTo + " len=" + len); + if (len <= downTo) { + // System.out.println(" final: offset=" + offset + " len=" + len + + // " dest=" + (downTo-len)); + arraycopy(b, offset, block, downTo - len, len); + break; + } else { + len -= downTo; + // System.out.println(" partial: offset=" + (offset + len) + + // " len=" + downTo + " dest=0"); + arraycopy(b, offset + len, block, 0, downTo); + blockIndex--; + block = blocks.get(blockIndex); + downTo = blockSize; + } + } + } + + /** + * Absolute copy bytes self to self, without changing the position. Note: this + * cannot "grow" the bytes, so must only call it on already written parts. + */ + public void copyBytes(long src, long dest, int len) { + // System.out.println("BS.copyBytes src=" + src + " dest=" + dest + " len=" + // + len); + assert src < dest; + + // Note: weird: must go "backwards" because copyBytes + // calls us with overlapping src/dest. If we + // go forwards then we overwrite bytes before we can + // copy them: + + /* + * int blockIndex = src >> blockBits; int upto = src & blockMask; byte[] + * block = blocks.get(blockIndex); while (len > 0) { int chunk = blockSize - + * upto; System.out.println(" cycle: chunk=" + chunk + " len=" + len); if + * (len <= chunk) { writeBytes(dest, block, upto, len); break; } else { + * writeBytes(dest, block, upto, chunk); blockIndex++; block = + * blocks.get(blockIndex); upto = 0; len -= chunk; dest += chunk; } } + */ + + long end = src + len; + + int blockIndex = (int) (end >> blockBits); + int downTo = (int) (end & blockMask); + if (downTo == 0) { + blockIndex--; + downTo = blockSize; + } + ByteArray block = blocks.get(blockIndex); + + while (len > 0) { + // System.out.println(" cycle downTo=" + downTo); + if (len <= downTo) { + // System.out.println(" finish"); + writeBytes(dest, block, downTo - len, len); + break; + } else { + // System.out.println(" partial"); + len -= downTo; + writeBytes(dest + len, block, 0, downTo); + blockIndex--; + block = blocks.get(blockIndex); + downTo = blockSize; + } + } + } + + /** + * Writes an int at the absolute position without changing the current + * pointer. + */ + public void writeInt(long pos, int value) { + int blockIndex = (int) (pos >> blockBits); + int upto = (int) (pos & blockMask); + ByteArray block = blocks.get(blockIndex); + int shift = 24; + for (int i = 0; i < 4; i++) { + block.put(upto++, (byte) (value >> shift)); + shift -= 8; + if (upto == blockSize) { + upto = 0; + blockIndex++; + block = blocks.get(blockIndex); + } + } + } + + /** Reverse from srcPos, inclusive, to destPos, inclusive. */ + public void reverse(long srcPos, long destPos) { + assert srcPos < destPos; + assert destPos < getPosition(); + // System.out.println("reverse src=" + srcPos + " dest=" + destPos); + + int srcBlockIndex = (int) (srcPos >> blockBits); + int src = (int) (srcPos & blockMask); + ByteArray srcBlock = blocks.get(srcBlockIndex); + + int destBlockIndex = (int) (destPos >> blockBits); + int dest = (int) (destPos & blockMask); + ByteArray destBlock = blocks.get(destBlockIndex); + // System.out.println(" srcBlock=" + srcBlockIndex + " destBlock=" + + // destBlockIndex); + + int limit = (int) (destPos - srcPos + 1) / 2; + for (int i = 0; i < limit; i++) { + // System.out.println(" cycle src=" + src + " dest=" + dest); + byte b = srcBlock.get(src); + srcBlock.put(src, destBlock.get(dest)); + destBlock.put(dest, b); + src++; + if (src == blockSize) { + srcBlockIndex++; + srcBlock = blocks.get(srcBlockIndex); + // System.out.println(" set destBlock=" + destBlock + " srcBlock=" + + // srcBlock); + src = 0; + } + + dest--; + if (dest == -1) { + destBlockIndex--; + destBlock = blocks.get(destBlockIndex); + // System.out.println(" set destBlock=" + destBlock + " srcBlock=" + + // srcBlock); + dest = blockSize - 1; + } + } + } + + public void skipBytes(int len) { + while (len > 0) { + int chunk = blockSize - nextWrite; + if (len <= chunk) { + nextWrite += len; + break; + } else { + len -= chunk; + current = factory.newByteArray(blockSize); + blocks.add(current); + nextWrite = 0; + } + } + } + + public long getPosition() { + return ((long) blocks.size() - 1) * blockSize + nextWrite; + } + + /** + * Pos must be less than the max position written so far! Ie, you cannot + * "grow" the file with this! + */ + public void truncate(long newLen) { + assert newLen <= getPosition(); + assert newLen >= 0; + int blockIndex = (int) (newLen >> blockBits); + nextWrite = (int) (newLen & blockMask); + if (nextWrite == 0) { + blockIndex--; + nextWrite = blockSize; + } + blocks.subList(blockIndex + 1, blocks.size()).clear(); + if (newLen == 0) { + current = null; + } else { + current = blocks.get(blockIndex); + } + assert newLen == getPosition(); + } + + public void finish() { + if (current != null) { + ByteArray lastBuffer = factory.newByteArray(nextWrite); + arraycopy(current, 0, lastBuffer, 0, nextWrite); + blocks.set(blocks.size() - 1, lastBuffer); + current = null; + } + } + + /** Writes all of our bytes to the target {@link DataOutput}. */ + public void writeTo(DataOutput out) throws IOException { + for (ByteArray block : blocks) { + block.writeBytes(out, 0, block.length()); + } + } + + public FST.BytesReader getForwardReader() { + if (blocks.size() == 1) { + return new ForwardBytesReaderByteArray(blocks.get(0)); + } + return new FST.BytesReader() { + private ByteArray current; + private int nextBuffer; + private int nextRead = blockSize; + + @Override + public byte readByte() { + if (nextRead == blockSize) { + current = blocks.get(nextBuffer++); + nextRead = 0; + } + return current.get(nextRead++); + } + + @Override + public void skipBytes(int count) { + setPosition(getPosition() + count); + } + + @Override + public void readBytes(byte[] b, int offset, int len) { + while (len > 0) { + int chunkLeft = blockSize - nextRead; + if (len <= chunkLeft) { + arraycopy(current, nextRead, b, offset, len); + nextRead += len; + break; + } else { + if (chunkLeft > 0) { + arraycopy(current, nextRead, b, offset, chunkLeft); + offset += chunkLeft; + len -= chunkLeft; + } + current = blocks.get(nextBuffer++); + nextRead = 0; + } + } + } + + @Override + public long getPosition() { + return ((long) nextBuffer - 1) * blockSize + nextRead; + } + + @Override + public void setPosition(long pos) { + int bufferIndex = (int) (pos >> blockBits); + nextBuffer = bufferIndex + 1; + current = blocks.get(bufferIndex); + nextRead = (int) (pos & blockMask); + assert getPosition() == pos; + } + + @Override + public boolean reversed() { + return false; + } + }; + } + + public FST.BytesReader getReverseReader() { + return getReverseReader(true); + } + + FST.BytesReader getReverseReader(boolean allowSingle) { + if (allowSingle && blocks.size() == 1) { + return new ReverseBytesReaderByteArray(blocks.get(0)); + } + return new FST.BytesReader() { + private ByteArray current = blocks.size() == 0 ? null : blocks.get(0); + private int nextBuffer = -1; + private int nextRead = 0; + + @Override + public byte readByte() { + if (nextRead == -1) { + current = blocks.get(nextBuffer--); + nextRead = blockSize - 1; + } + return current.get(nextRead--); + } + + @Override + public void skipBytes(int count) { + setPosition(getPosition() - count); + } + + @Override + public void readBytes(byte[] b, int offset, int len) { + for (int i = 0; i < len; i++) { + b[offset + i] = readByte(); + } + } + + @Override + public long getPosition() { + return ((long) nextBuffer + 1) * blockSize + nextRead; + } + + @Override + public void setPosition(long pos) { + // NOTE: a little weird because if you + // setPosition(0), the next byte you read is + // bytes[0] ... but I would expect bytes[-1] (ie, + // EOF)...? + int bufferIndex = (int) (pos >> blockBits); + nextBuffer = bufferIndex - 1; + current = blocks.get(bufferIndex); + nextRead = (int) (pos & blockMask); + assert getPosition() == pos : "pos=" + pos + " getPos()=" + getPosition(); + } + + @Override + public boolean reversed() { + return true; + } + }; + } + + static class ForwardBytesReaderByteArray extends FST.BytesReader { + private final ByteArray bytes; + private int pos; + + public ForwardBytesReaderByteArray(ByteArray bytes) { + this.bytes = bytes; + } + + @Override + public byte readByte() { + return bytes.get(pos++); + } + + @Override + public void readBytes(byte[] b, int offset, int len) { + arraycopy(bytes, pos, b, offset, len); + pos += len; + } + + @Override + public void skipBytes(int count) { + pos += count; + } + + @Override + public long getPosition() { + return pos; + } + + @Override + public void setPosition(long pos) { + this.pos = (int) pos; + } + + @Override + public boolean reversed() { + return false; + } + } + + static class ReverseBytesReaderByteArray extends FST.BytesReader { + private final ByteArray bytes; + private int pos; + + public ReverseBytesReaderByteArray(ByteArray bytes) { + this.bytes = bytes; + } + + @Override + public byte readByte() { + return bytes.get(pos--); + } + + @Override + public void readBytes(byte[] b, int offset, int len) { + for (int i = 0; i < len; i++) { + b[offset + i] = bytes.get(pos--); + } + } + + @Override + public void skipBytes(int count) { + pos -= count; + } + + @Override + public long getPosition() { + return pos; + } + + @Override + public void setPosition(long pos) { + this.pos = (int) pos; + } + + @Override + public boolean reversed() { + return true; + } + } + + public static void arraycopy(ByteArray src, int srcOffset, byte[] dest, int destOffset, int len) { + src.get(srcOffset, dest, destOffset, len); + } + + public static void arraycopy(ByteArray src, int srcOffset, ByteArray dest, int destOffset, int len) { + src.copy(srcOffset, dest, destOffset, len); + } + + public static void arraycopy(byte[] src, int srcOffset, ByteArray dest, int destOffset, int len) { + dest.put(destOffset, src, srcOffset, len); + } + +} http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/393c9f53/blur-util/src/main/java/org/apache/blur/utils/BlurConstants.java ---------------------------------------------------------------------- diff --git a/blur-util/src/main/java/org/apache/blur/utils/BlurConstants.java b/blur-util/src/main/java/org/apache/blur/utils/BlurConstants.java index 7e20fe1..a6d7824 100644 --- a/blur-util/src/main/java/org/apache/blur/utils/BlurConstants.java +++ b/blur-util/src/main/java/org/apache/blur/utils/BlurConstants.java @@ -129,6 +129,7 @@ public class BlurConstants { public static final String BLUR_CONTROLLER_THRIFT_ACCEPT_QUEUE_SIZE_PER_THREAD = "blur.controller.thrift.accept.queue.size.per.thread"; public static final String BLUR_CLIENTPOOL_CLIENT_CLOSE_THRESHOLD = "blur.clientpool.client.close.threshold"; public static final String BLUR_CLIENTPOOL_CLIENT_CLEAN_FREQUENCY = "blur.clientpool.client.clean.frequency"; + public static final String BLUR_LUCENE_FST_BYTEARRAY_FACTORY = "blur.lucene.fst.bytearray.factory"; public static final String BLUR_THRIFT_MAX_FRAME_SIZE = "blur.thrift.max.frame.size"; http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/393c9f53/blur-util/src/main/resources/blur-default.properties ---------------------------------------------------------------------- diff --git a/blur-util/src/main/resources/blur-default.properties b/blur-util/src/main/resources/blur-default.properties index 9a0e60c..10f8f77 100644 --- a/blur-util/src/main/resources/blur-default.properties +++ b/blur-util/src/main/resources/blur-default.properties @@ -184,6 +184,9 @@ blur.shard.blurindex.class= # Defines the blur read interceptor class that can mask data from query results as well as data fetches. blur.shard.read.interceptor= +# Defines the byte array factory class that blur will use to manage the FST trees in Lucene (extends org.apache.blur.lucene.fst.ByteArrayFactory). +blur.lucene.fst.bytearray.factory= + ### Controller Server Configuration
