This is an automated email from the ASF dual-hosted git repository.
thomasm pushed a commit to branch OAK-10341
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git
The following commit(s) were added to refs/heads/OAK-10341 by this push:
new 3e2bc7376f OAK-10341 TreeStore
3e2bc7376f is described below
commit 3e2bc7376fd35713e182004f8ad2a55a5b7b0d80
Author: Thomas Mueller <[email protected]>
AuthorDate: Fri Jul 7 13:54:47 2023 +0200
OAK-10341 TreeStore
---
.../indexer/document/DocumentStoreIndexerBase.java | 2 +-
.../oak/index/indexer/document/tree/TreeStore.java | 96 ++++++++++++++++++++--
.../indexer/document/tree/TreeStoreNodeState.java | 4 +-
.../indexer/document/tree/store/StoreBuilder.java | 1 +
4 files changed, 94 insertions(+), 9 deletions(-)
diff --git
a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/DocumentStoreIndexerBase.java
b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/DocumentStoreIndexerBase.java
index a8650c5fbd..5370ffc224 100644
---
a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/DocumentStoreIndexerBase.java
+++
b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/DocumentStoreIndexerBase.java
@@ -231,7 +231,7 @@ public abstract class DocumentStoreIndexerBase implements
Closeable {
public void reindexUsingTreeStore() throws CommitFailedException,
IOException {
NodeStateEntryReader reader = new
NodeStateEntryReader(indexHelper.getGCBlobStore());
- TreeStore treeStore = new TreeStore(new File("target"), reader);
+ TreeStore treeStore = new TreeStore(new File("target/treeStore"),
reader);
// TODO this is mostly a copy of reindex()
diff --git
a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStore.java
b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStore.java
index 6d63f0e329..3c5eb588c4 100644
---
a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStore.java
+++
b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStore.java
@@ -18,9 +18,12 @@
*/
package org.apache.jackrabbit.oak.index.indexer.document.tree;
+import java.io.BufferedReader;
import java.io.Closeable;
import java.io.File;
+import java.io.FileReader;
import java.io.IOException;
+import java.nio.charset.StandardCharsets;
import java.util.Iterator;
import java.util.Map.Entry;
@@ -31,17 +34,87 @@ import
org.apache.jackrabbit.oak.index.indexer.document.flatfile.NodeStateEntryR
import org.apache.jackrabbit.oak.index.indexer.document.tree.store.Session;
import org.apache.jackrabbit.oak.index.indexer.document.tree.store.Store;
import
org.apache.jackrabbit.oak.index.indexer.document.tree.store.StoreBuilder;
+import org.apache.jackrabbit.oak.index.indexer.document.tree.store.utils.Cache;
+import org.apache.jackrabbit.oak.plugins.memory.EmptyNodeState;
+import org.apache.jackrabbit.oak.spi.blob.MemoryBlobStore;
import org.apache.jackrabbit.oak.spi.state.NodeState;
public class TreeStore implements Iterable<NodeStateEntry>, Closeable {
+ public static void main(String... args) throws IOException {
+ String dir = args[0];
+ MemoryBlobStore blobStore = new MemoryBlobStore();
+ NodeStateEntryReader entryReader = new NodeStateEntryReader(blobStore);
+ TreeStore treeStore = new TreeStore(new File(dir), entryReader);
+ Session session = treeStore.session;
+ Store store = treeStore.store;
+ if (store.keySet().isEmpty()) {
+ session.init();
+ String fileName = args[1];
+ BufferedReader lineReader = new BufferedReader(
+ new FileReader(fileName, StandardCharsets.UTF_8));
+ int count = 0;
+ long start = System.nanoTime();
+ while (true) {
+ String line = lineReader.readLine();
+ if (line == null) {
+ break;
+ }
+ count++;
+ if (count % 1000000 == 0) {
+ long time = System.nanoTime() - start;
+ System.out.println(count + " " + (time / count) + "
ns/entry");
+ }
+ int index = line.indexOf('|');
+ if (index < 0) {
+ throw new IllegalArgumentException("| is missing: " +
line);
+ }
+ String path = line.substring(0, index);
+ String value = line.substring(index + 1);
+ session.put(path, value);
+
+ if (!path.equals("/")) {
+ String nodeName = PathUtils.getName(path);
+ String parentPath = PathUtils.getParentPath(path);
+ session.put(parentPath + "\t" + nodeName, "");
+ }
+
+ }
+ lineReader.close();
+ session.flush();
+ store.close();
+ }
+ Iterator<NodeStateEntry> it = treeStore.iterator();
+ long nodeCount = 0;
+ long childNodeCount = 0;
+ long start = System.nanoTime();
+ while (it.hasNext()) {
+ NodeStateEntry e = it.next();
+ childNodeCount +=
e.getNodeState().getChildNodeCount(Long.MAX_VALUE);
+ nodeCount++;
+ if (nodeCount % 1000000 == 0) {
+ long time = System.nanoTime() - start;
+ System.out.println("Node count: " + nodeCount +
+ " child node count: " + childNodeCount +
+ " speed " + (time / nodeCount) + " ns/entry");
+ }
+ }
+ System.out.println("Node count: " + nodeCount + " Child node count: "
+ childNodeCount);
+ }
+
private final Store store;
private final Session session;
private final NodeStateEntryReader entryReader;
+ private final Cache<String, NodeState> nodeStateCache = new Cache<>(100);
public TreeStore(File directory, NodeStateEntryReader entryReader) {
this.entryReader = entryReader;
- this.store = StoreBuilder.build("type=file\n" + "dir=" +
directory.getAbsolutePath());
+ String storeConfig = System.getProperty("oak.treeStoreConfig",
+ "type=file\n" +
+ "cacheSizeMB=4096\n" +
+ "maxFileSize=64000000\n" +
+ "dir=" + directory.getAbsolutePath());
+ this.store = StoreBuilder.build(storeConfig);
this.session = new Session(store);
}
@@ -69,6 +142,7 @@ public class TreeStore implements Iterable<NodeStateEntry>,
Closeable {
continue;
}
current = getNodeStateEntry(e.getKey(), e.getValue());
+ return;
}
current = null;
}
@@ -97,18 +171,30 @@ public class TreeStore implements
Iterable<NodeStateEntry>, Closeable {
}
NodeState getNodeState(String path) {
+ NodeState result = nodeStateCache.get(path);
+ if (result != null) {
+ return result;
+ }
String value = session.get(path);
if (value == null || value.isEmpty()) {
- throw new IllegalArgumentException(path);
+ result = EmptyNodeState.EMPTY_NODE;
+ } else {
+ result = getNodeState(path, value);
}
- return getNodeState(path, value);
+ nodeStateCache.put(path, result);
+ return result;
}
NodeState getNodeState(String path, String value) {
+ NodeState result = nodeStateCache.get(path);
+ if (result != null) {
+ return result;
+ }
String line = path + "|" + value;
NodeStateEntry entry = entryReader.read(line);
- NodeState wrapped = new TreeStoreNodeState(entry.getNodeState(), path,
this);
- return wrapped;
+ result = new TreeStoreNodeState(entry.getNodeState(), path, this);
+ nodeStateCache.put(path, result);
+ return result;
}
/**
diff --git
a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStoreNodeState.java
b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStoreNodeState.java
index b6afccffdb..9a86d30e76 100644
---
a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStoreNodeState.java
+++
b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStoreNodeState.java
@@ -130,9 +130,7 @@ public class TreeStoreNodeState implements NodeState {
@NotNull
@Override
public NodeState getChildNode(@NotNull String name) throws
IllegalArgumentException {
- // TODO
- return null;
-// String path = PathUtils.concat(path, name);
+ return treeStore.getNodeState(PathUtils.concat(path, name));
}
@Override
diff --git
a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/StoreBuilder.java
b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/StoreBuilder.java
index ccab3e112c..f6aeced144 100644
---
a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/StoreBuilder.java
+++
b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/StoreBuilder.java
@@ -41,6 +41,7 @@ public class StoreBuilder {
if (config == null || config.isEmpty()) {
return new MemoryStore(new Properties());
}
+ config = config.replace(' ', '\n');
Properties prop = new Properties();
try {
prop.load(new StringReader(config));
