Author: jukka
Date: Thu Jul 11 14:43:59 2013
New Revision: 1502238
URL: http://svn.apache.org/r1502238
Log:
OAK-895: Random access for Lucene index binaries
Split the Lucene index files to arrays of smaller blobs so we can read and
modify them in a more granular manner.
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/core/SecureNodeBuilder.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/memory/MemoryNodeBuilder.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/spi/state/NodeBuilder.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/spi/state/ReadOnlyBuilder.java
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditorContext.java
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/OakDirectory.java
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/core/SecureNodeBuilder.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/core/SecureNodeBuilder.java?rev=1502238&r1=1502237&r2=1502238&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/core/SecureNodeBuilder.java
(original)
+++
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/core/SecureNodeBuilder.java
Thu Jul 11 14:43:59 2013
@@ -24,12 +24,16 @@ import static org.apache.jackrabbit.oak.
import static org.apache.jackrabbit.oak.api.Type.NAME;
import static org.apache.jackrabbit.oak.api.Type.NAMES;
+import java.io.IOException;
+import java.io.InputStream;
+
import javax.annotation.CheckForNull;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import com.google.common.base.Predicate;
+import org.apache.jackrabbit.oak.api.Blob;
import org.apache.jackrabbit.oak.api.PropertyState;
import org.apache.jackrabbit.oak.api.Type;
import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
@@ -246,6 +250,11 @@ class SecureNodeBuilder implements NodeB
}
}
+ @Override
+ public Blob createBlob(InputStream stream) throws IOException {
+ return builder.createBlob(stream);
+ }
+
//------------------------------------------------------< inner classes
>---
/**
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/memory/MemoryNodeBuilder.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/memory/MemoryNodeBuilder.java?rev=1502238&r1=1502237&r2=1502238&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/memory/MemoryNodeBuilder.java
(original)
+++
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/memory/MemoryNodeBuilder.java
Thu Jul 11 14:43:59 2013
@@ -25,13 +25,19 @@ import static org.apache.jackrabbit.oak.
import static org.apache.jackrabbit.oak.api.Type.NAMES;
import static
org.apache.jackrabbit.oak.plugins.memory.EmptyNodeState.EMPTY_NODE;
+import java.io.IOException;
+import java.io.InputStream;
+
import javax.annotation.Nonnull;
+import org.apache.jackrabbit.oak.api.Blob;
import org.apache.jackrabbit.oak.api.PropertyState;
import org.apache.jackrabbit.oak.api.Type;
import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
import org.apache.jackrabbit.oak.spi.state.NodeState;
+import com.google.common.io.ByteStreams;
+
/**
* In-memory node state builder.
* <p>
@@ -355,6 +361,15 @@ public class MemoryNodeBuilder implement
return this;
}
+ @Override
+ public Blob createBlob(InputStream stream) throws IOException {
+ try {
+ return new ArrayBasedBlob(ByteStreams.toByteArray(stream));
+ } finally {
+ stream.close();
+ }
+ }
+
/**
* @return path of this builder. For debugging purposes only
*/
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/spi/state/NodeBuilder.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/spi/state/NodeBuilder.java?rev=1502238&r1=1502237&r2=1502238&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/spi/state/NodeBuilder.java
(original)
+++
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/spi/state/NodeBuilder.java
Thu Jul 11 14:43:59 2013
@@ -16,9 +16,13 @@
*/
package org.apache.jackrabbit.oak.spi.state;
+import java.io.IOException;
+import java.io.InputStream;
+
import javax.annotation.CheckForNull;
import javax.annotation.Nonnull;
+import org.apache.jackrabbit.oak.api.Blob;
import org.apache.jackrabbit.oak.api.PropertyState;
import org.apache.jackrabbit.oak.api.Type;
@@ -339,4 +343,6 @@ public interface NodeBuilder {
@Nonnull
NodeBuilder removeProperty(String name);
+ Blob createBlob(InputStream stream) throws IOException;
+
}
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/spi/state/ReadOnlyBuilder.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/spi/state/ReadOnlyBuilder.java?rev=1502238&r1=1502237&r2=1502238&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/spi/state/ReadOnlyBuilder.java
(original)
+++
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/spi/state/ReadOnlyBuilder.java
Thu Jul 11 14:43:59 2013
@@ -16,9 +16,13 @@
*/
package org.apache.jackrabbit.oak.spi.state;
+import java.io.IOException;
+import java.io.InputStream;
+
import javax.annotation.CheckForNull;
import javax.annotation.Nonnull;
+import org.apache.jackrabbit.oak.api.Blob;
import org.apache.jackrabbit.oak.api.PropertyState;
import org.apache.jackrabbit.oak.api.Type;
@@ -168,4 +172,9 @@ public class ReadOnlyBuilder implements
throw unsupported();
}
+ @Override
+ public Blob createBlob(InputStream stream) throws IOException {
+ throw unsupported();
+ }
+
}
Modified:
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditorContext.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditorContext.java?rev=1502238&r1=1502237&r2=1502238&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditorContext.java
(original)
+++
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditorContext.java
Thu Jul 11 14:43:59 2013
@@ -66,8 +66,7 @@ public class LuceneIndexEditorContext {
throws IOException {
String path = getString(definition, PERSISTENCE_PATH);
if (path == null) {
- return new OakDirectory(
- definition.child(INDEX_DATA_CHILD_NAME));
+ return new OakDirectory(definition.child(INDEX_DATA_CHILD_NAME));
} else {
// try {
File file = new File(path);
Modified:
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/OakDirectory.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/OakDirectory.java?rev=1502238&r1=1502237&r2=1502238&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/OakDirectory.java
(original)
+++
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/OakDirectory.java
Thu Jul 11 14:43:59 2013
@@ -16,13 +16,26 @@
*/
package org.apache.jackrabbit.oak.plugins.index.lucene;
-import static org.apache.jackrabbit.oak.api.Type.BINARY;
+import static com.google.common.base.Preconditions.checkElementIndex;
+import static com.google.common.base.Preconditions.checkNotNull;
+import static com.google.common.base.Preconditions.checkPositionIndexes;
+import static com.google.common.base.Preconditions.checkState;
+import static com.google.common.collect.Lists.newArrayList;
+import static org.apache.jackrabbit.JcrConstants.JCR_DATA;
+import static org.apache.jackrabbit.JcrConstants.JCR_LASTMODIFIED;
+import static org.apache.jackrabbit.oak.api.Type.BINARIES;
+import java.io.ByteArrayInputStream;
+import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.util.Collection;
+import java.util.List;
import com.google.common.collect.Iterables;
+import com.google.common.io.ByteStreams;
+
+import org.apache.jackrabbit.oak.api.Blob;
import org.apache.jackrabbit.oak.api.PropertyState;
import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
import org.apache.lucene.store.Directory;
@@ -62,30 +75,31 @@ class OakDirectory extends Directory {
@Override
public long fileLength(String name) throws IOException {
- if (!fileExists(name)) {
- return 0;
- }
-
- NodeBuilder fileBuilder = directoryBuilder.child(name);
- PropertyState property = fileBuilder.getProperty("jcr:data");
- if (property == null || property.isArray()) {
- return 0;
+ NodeBuilder file = directoryBuilder.getChildNode(name);
+ OakIndexInput input = new OakIndexInput(name, file);
+ try {
+ return input.length();
+ } finally {
+ input.close();
}
-
- return property.size();
}
@Override
public IndexOutput createOutput(String name, IOContext context)
throws IOException {
- return new OakIndexOutput(name);
+ return new OakIndexOutput(name, directoryBuilder.child(name));
}
@Override
public IndexInput openInput(String name, IOContext context)
throws IOException {
- return new OakIndexInput(name);
+ NodeBuilder file = directoryBuilder.getChildNode(name);
+ if (file.exists()) {
+ return new OakIndexInput(name, file);
+ } else {
+ throw new FileNotFoundException(name);
+ }
}
@Override
@@ -98,88 +112,212 @@ class OakDirectory extends Directory {
// do nothing
}
- protected byte[] readFile(String name) throws IOException {
- if (!fileExists(name)) {
- return new byte[0];
+ private static final int BLOB_SIZE = 4092;
+
+ private static class OakIndexFile {
+
+ private final String name;
+
+ private final NodeBuilder file;
+
+ private long position = 0;
+
+ private long length;
+
+ private final List<Blob> data;
+
+ private boolean dataModified = false;
+
+ private int index = -1;
+
+ private final byte[] blob = new byte[BLOB_SIZE];
+
+ private boolean blobModified = false;
+
+ public OakIndexFile(String name, NodeBuilder file) {
+ this.name = name;
+ this.file = file;
+
+ PropertyState property = file.getProperty(JCR_DATA);
+ if (property != null && property.getType() == BINARIES) {
+ this.data = newArrayList(property.getValue(BINARIES));
+ } else {
+ this.data = newArrayList();
+ }
+
+ this.length = data.size() * BLOB_SIZE;
+ if (!data.isEmpty()) {
+ Blob last = data.get(data.size() - 1);
+ this.length -= BLOB_SIZE - last.length();
+ }
}
- NodeBuilder fileBuilder = directoryBuilder.child(name);
- PropertyState property = fileBuilder.getProperty("jcr:data");
- if (property == null || property.isArray()) {
- return new byte[0];
+ private OakIndexFile(OakIndexFile that) {
+ this.name = that.name;
+ this.file = that.file;
+
+ this.position = that.position;
+ this.length = that.length;
+ this.data = newArrayList(that.data);
+ this.dataModified = that.dataModified;
+ }
+
+ private void loadBlob(int i) throws IOException {
+ checkElementIndex(i, data.size());
+ if (index != i) {
+ flushBlob();
+ checkState(!blobModified);
+
+ int n = (int) Math.min(BLOB_SIZE, length - i * BLOB_SIZE);
+ InputStream stream = data.get(i).getNewStream();
+ try {
+ ByteStreams.readFully(stream, blob, 0, n);
+ } finally {
+ stream.close();
+ }
+ index = i;
+ }
}
- InputStream stream = property.getValue(BINARY).getNewStream();
- try {
- byte[] buffer = new byte[(int) property.size()];
+ private void flushBlob() throws IOException {
+ if (blobModified) {
+ System.out.format("%s flush blob %d%n", name, index);
+ int n = (int) Math.min(BLOB_SIZE, length - index * BLOB_SIZE);
+ Blob b = file.createBlob(new ByteArrayInputStream(blob, 0, n));
+ if (index < data.size()) {
+ data.set(index, b);
+ } else {
+ checkState(index == data.size());
+ data.add(b);
+ }
+ dataModified = true;
+ blobModified = false;
+ }
+ }
+
+ public void seek(long pos) throws IOException {
+ // seek() may be called with pos == length
+ // see https://issues.apache.org/jira/browse/LUCENE-1196
+ if (pos < 0 || pos > length) {
+ throw new IOException("Invalid seek request");
+ } else {
+ position = pos;
+ }
+ }
+
+ public void readBytes(byte[] b, int offset, int len)
+ throws IOException {
+ checkPositionIndexes(offset, offset + len, checkNotNull(b).length);
+
+ if (len < 0 || position + len > length) {
+ throw new IOException("Invalid byte range request");
+ }
+
+ int i = (int) (position / BLOB_SIZE);
+ int o = (int) (position % BLOB_SIZE);
+ while (len > 0) {
+ loadBlob(i);
+
+ int l = Math.min(len, BLOB_SIZE - o);
+ System.arraycopy(blob, o, b, offset, l);
+
+ offset += l;
+ len -= l;
+ position += l;
+
+ i++;
+ o = 0;
+ }
+ }
- int size = 0;
- do {
- int n = stream.read(buffer, size, buffer.length - size);
- if (n == -1) {
- throw new IOException(
- "Unexpected end of index file: " + name);
+ public void writeBytes(byte[] b, int offset, int len)
+ throws IOException {
+ int i = (int) (position / BLOB_SIZE);
+ int o = (int) (position % BLOB_SIZE);
+ while (len > 0) {
+ int l = Math.min(len, BLOB_SIZE - o);
+
+ if (index != i) {
+ if (o > 0 || (l < BLOB_SIZE && position + l < length)) {
+ loadBlob(i);
+ } else {
+ flushBlob();
+ index = i;
+ }
}
- size += n;
- } while (size < buffer.length);
+ System.arraycopy(b, offset, blob, o, l);
+ blobModified = true;
- return buffer;
- } finally {
- stream.close();
+ offset += l;
+ len -= l;
+ position += l;
+ length = Math.max(length, position);
+
+ i++;
+ o = 0;
+ }
}
- }
- private final class OakIndexInput extends IndexInput {
+ public void flush() throws IOException {
+ flushBlob();
+ if (dataModified) {
+ file.setProperty(JCR_LASTMODIFIED, System.currentTimeMillis());
+ file.setProperty(JCR_DATA, data, BINARIES);
+ dataModified = false;
+ }
+ }
- private final byte[] data;
+ @Override
+ public String toString() {
+ return name;
+ }
- private int position;
+ }
+
+ private static class OakIndexInput extends IndexInput {
+
+ private final OakIndexFile file;
- public OakIndexInput(String name) throws IOException {
+ public OakIndexInput(String name, NodeBuilder file) {
super(name);
- this.data = readFile(name);
- this.position = 0;
+ this.file = new OakIndexFile(name, file);
+ }
+
+ private OakIndexInput(OakIndexInput that) {
+ super(that.toString());
+ this.file = new OakIndexFile(that.file);
}
@Override
- public void readBytes(byte[] b, int offset, int len)
- throws IOException {
- if (len < 0 || position + len > data.length) {
- throw new IOException("Invalid byte range request");
- } else {
- System.arraycopy(data, position, b, offset, len);
- position += len;
- }
+ public OakIndexInput clone() {
+ return new OakIndexInput(this);
+ }
+
+ @Override
+ public void readBytes(byte[] b, int o, int n) throws IOException {
+ file.readBytes(b, o, n);
}
@Override
public byte readByte() throws IOException {
- if (position >= data.length) {
- throw new IOException("Invalid byte range request");
- } else {
- return data[position++];
- }
+ byte[] b = new byte[1];
+ readBytes(b, 0, 1);
+ return b[0];
}
@Override
public void seek(long pos) throws IOException {
- //seek() may be called with pos == data.length
- //see https://issues.apache.org/jira/browse/LUCENE-1196
- if (pos < 0 || pos > data.length) {
- throw new IOException("Invalid seek request");
- } else {
- position = (int) pos;
- }
+ file.seek(pos);
}
@Override
public long length() {
- return data.length;
+ return file.length;
}
@Override
public long getFilePointer() {
- return position;
+ return file.position;
}
@Override
@@ -191,77 +329,48 @@ class OakDirectory extends Directory {
private final class OakIndexOutput extends IndexOutput {
- private final String name;
-
- private byte[] buffer;
-
- private int size;
+ private final OakIndexFile file;
- private int position;
-
- public OakIndexOutput(String name) throws IOException {
- this.name = name;
- this.buffer = readFile(name);
- this.size = buffer.length;
- this.position = 0;
+ public OakIndexOutput(String name, NodeBuilder file) throws
IOException {
+ this.file = new OakIndexFile(name, file);
}
@Override
public long length() {
- return size;
+ return file.length;
}
@Override
public long getFilePointer() {
- return position;
+ return file.position;
}
@Override
public void seek(long pos) throws IOException {
- if (pos < 0 || pos > Integer.MAX_VALUE) {
- throw new IOException("Invalid file position: " + pos);
- }
- this.position = (int) pos;
+ file.seek(pos);
}
@Override
- public void writeBytes(byte[] b, int offset, int length) {
- while (position + length > buffer.length) {
- byte[] tmp = new byte[Math.max(4096, buffer.length * 2)];
- System.arraycopy(buffer, 0, tmp, 0, size);
- buffer = tmp;
- }
-
- System.arraycopy(b, offset, buffer, position, length);
-
- position += length;
- if (position > size) {
- size = position;
- }
+ public void writeBytes(byte[] b, int offset, int length)
+ throws IOException {
+ file.writeBytes(b, offset, length);
}
@Override
- public void writeByte(byte b) {
+ public void writeByte(byte b) throws IOException {
writeBytes(new byte[] { b }, 0, 1);
}
@Override
public void flush() throws IOException {
- byte[] data = buffer;
- if (data.length > size) {
- data = new byte[size];
- System.arraycopy(buffer, 0, data, 0, size);
- }
-
- NodeBuilder fileBuilder = directoryBuilder.child(name);
- fileBuilder.setProperty("jcr:lastModified",
System.currentTimeMillis());
- fileBuilder.setProperty("jcr:data", data);
+ file.flush();
}
@Override
public void close() throws IOException {
flush();
}
+
}
}
