Author: chetanm
Date: Wed Jan 3 09:48:29 2018
New Revision: 1819936
URL: http://svn.apache.org/viewvc?rev=1819936&view=rev
Log:
OAK-7115 - Store NodeState json in bytes when storing in in-memory queue
Added:
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/NodeStateHolder.java
- copied, changed from r1819421,
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/NodeStateEntryHolder.java
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/SimpleNodeStateHolder.java
- copied, changed from r1819421,
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/NodeStateEntryHolder.java
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/StateInBytesHolder.java
(contents, props changed)
- copied, changed from r1819421,
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/NodeStateEntryHolder.java
Removed:
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/NodeStateEntryHolder.java
Modified:
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/NodeStateEntrySorter.java
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/TraverseWithSortStrategy.java
Modified:
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/NodeStateEntrySorter.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/NodeStateEntrySorter.java?rev=1819936&r1=1819935&r2=1819936&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/NodeStateEntrySorter.java
(original)
+++
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/NodeStateEntrySorter.java
Wed Jan 3 09:48:29 2018
@@ -89,9 +89,9 @@ public class NodeStateEntrySorter {
log.info("Sorting with memory {} (estimated {})",
humanReadableByteCount(memory), humanReadableByteCount(estimatedMemory));
Stopwatch w = Stopwatch.createStarted();
- Comparator<NodeStateEntryHolder> comparator = (e1, e2) ->
pathComparator.compare(e1.pathElements, e2.pathElements);
- Function<String, NodeStateEntryHolder> func1 = (line) -> line == null
? null : new NodeStateEntryHolder(line);
- Function<NodeStateEntryHolder, String> func2 = holder -> holder ==
null ? null : holder.line;
+ Comparator<NodeStateHolder> comparator = (e1, e2) ->
pathComparator.compare(e1.getPathElements(), e2.getPathElements());
+ Function<String, NodeStateHolder> func1 = (line) -> line == null ?
null : new SimpleNodeStateHolder(line);
+ Function<NodeStateHolder, String> func2 = holder -> holder == null ?
null : holder.getLine();
List<File> sortedFiles = sortInBatch(memory, comparator, func1, func2);
@@ -111,8 +111,8 @@ public class NodeStateEntrySorter {
log.info("Sorting completed in {}", w);
}
- private void mergeSortedFiles(Comparator<NodeStateEntryHolder> comparator,
Function<String, NodeStateEntryHolder> func1,
- Function<NodeStateEntryHolder, String>
func2, List<File> sortedFiles) throws IOException {
+ private void mergeSortedFiles(Comparator<NodeStateHolder> comparator,
Function<String, NodeStateHolder> func1,
+ Function<NodeStateHolder, String> func2,
List<File> sortedFiles) throws IOException {
try(BufferedWriter writer = createWriter(sortedFile, useZip)) {
ExternalSort.mergeSortedFiles(sortedFiles,
writer,
@@ -127,9 +127,9 @@ public class NodeStateEntrySorter {
}
}
- private List<File> sortInBatch(long memory,
Comparator<NodeStateEntryHolder> comparator,
- Function<String, NodeStateEntryHolder>
func1,
- Function<NodeStateEntryHolder, String>
func2) throws IOException {
+ private List<File> sortInBatch(long memory, Comparator<NodeStateHolder>
comparator,
+ Function<String, NodeStateHolder> func1,
+ Function<NodeStateHolder, String> func2)
throws IOException {
if (useZip) {
try (BufferedReader reader = createReader(nodeStateFile, useZip)) {
return ExternalSort.sortInBatch(reader,
Copied:
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/NodeStateHolder.java
(from r1819421,
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/NodeStateEntryHolder.java)
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/NodeStateHolder.java?p2=jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/NodeStateHolder.java&p1=jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/NodeStateEntryHolder.java&r1=1819421&r2=1819936&rev=1819936&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/NodeStateEntryHolder.java
(original)
+++
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/NodeStateHolder.java
Wed Jan 3 09:48:29 2018
@@ -21,20 +21,11 @@ package org.apache.jackrabbit.oak.index.
import java.util.List;
-import static com.google.common.collect.ImmutableList.copyOf;
-import static org.apache.jackrabbit.oak.commons.PathUtils.elements;
-import static
org.apache.jackrabbit.oak.index.indexer.document.flatfile.NodeStateEntryWriter.getPath;
+public interface NodeStateHolder {
-class NodeStateEntryHolder {
- final String line;
- final List<String> pathElements;
+ List<String> getPathElements();
- public NodeStateEntryHolder(String line) {
- this(getPath(line), line);
- }
+ String getLine();
- public NodeStateEntryHolder(String path, String line) {
- this.pathElements = copyOf(elements(path));
- this.line = line;
- }
+ int getMemorySize();
}
Copied:
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/SimpleNodeStateHolder.java
(from r1819421,
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/NodeStateEntryHolder.java)
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/SimpleNodeStateHolder.java?p2=jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/SimpleNodeStateHolder.java&p1=jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/NodeStateEntryHolder.java&r1=1819421&r2=1819936&rev=1819936&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/NodeStateEntryHolder.java
(original)
+++
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/SimpleNodeStateHolder.java
Wed Jan 3 09:48:29 2018
@@ -21,20 +21,41 @@ package org.apache.jackrabbit.oak.index.
import java.util.List;
+import org.apache.jackrabbit.oak.commons.StringUtils;
+
import static com.google.common.collect.ImmutableList.copyOf;
import static org.apache.jackrabbit.oak.commons.PathUtils.elements;
import static
org.apache.jackrabbit.oak.index.indexer.document.flatfile.NodeStateEntryWriter.getPath;
-class NodeStateEntryHolder {
- final String line;
- final List<String> pathElements;
+class SimpleNodeStateHolder implements NodeStateHolder{
+ private final String line;
+ private final List<String> pathElements;
- public NodeStateEntryHolder(String line) {
- this(getPath(line), line);
+ public SimpleNodeStateHolder(String line) {
+ this.pathElements = copyOf(elements(getPath(line)));
+ this.line = line;
}
- public NodeStateEntryHolder(String path, String line) {
- this.pathElements = copyOf(elements(path));
- this.line = line;
+ @Override
+ public List<String> getPathElements() {
+ return pathElements;
+ }
+
+ /**
+ * Line here contains the path also
+ */
+ @Override
+ public String getLine() {
+ return line;
+ }
+
+ @Override
+ public int getMemorySize() {
+ int memoryUsed = 0;
+ for (String e : pathElements) {
+ memoryUsed += StringUtils.estimateMemoryUsage(e);
+ }
+ memoryUsed += StringUtils.estimateMemoryUsage(line);
+ return memoryUsed;
}
}
Copied:
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/StateInBytesHolder.java
(from r1819421,
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/NodeStateEntryHolder.java)
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/StateInBytesHolder.java?p2=jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/StateInBytesHolder.java&p1=jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/NodeStateEntryHolder.java&r1=1819421&r2=1819936&rev=1819936&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/NodeStateEntryHolder.java
(original)
+++
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/StateInBytesHolder.java
Wed Jan 3 09:48:29 2018
@@ -21,20 +21,41 @@ package org.apache.jackrabbit.oak.index.
import java.util.List;
+import org.apache.jackrabbit.oak.commons.StringUtils;
+
+import static com.google.common.base.Charsets.UTF_8;
import static com.google.common.collect.ImmutableList.copyOf;
import static org.apache.jackrabbit.oak.commons.PathUtils.elements;
-import static
org.apache.jackrabbit.oak.index.indexer.document.flatfile.NodeStateEntryWriter.getPath;
-class NodeStateEntryHolder {
- final String line;
- final List<String> pathElements;
+class StateInBytesHolder implements NodeStateHolder {
+ private final List<String> pathElements;
+ private final byte[] content;
- public NodeStateEntryHolder(String line) {
- this(getPath(line), line);
+ public StateInBytesHolder(String path, String line) {
+ this.pathElements = copyOf(elements(path));
+ this.content = line.getBytes(UTF_8);
}
- public NodeStateEntryHolder(String path, String line) {
- this.pathElements = copyOf(elements(path));
- this.line = line;
+ @Override
+ public List<String> getPathElements() {
+ return pathElements;
+ }
+
+ /**
+ * Line here does not include the path
+ */
+ @Override
+ public String getLine() {
+ return new String(content, UTF_8);
+ }
+
+ @Override
+ public int getMemorySize() {
+ int memoryUsed = 0;
+ for (String e : pathElements) {
+ memoryUsed += StringUtils.estimateMemoryUsage(e);
+ }
+ memoryUsed += content.length;
+ return memoryUsed;
}
}
Propchange:
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/StateInBytesHolder.java
------------------------------------------------------------------------------
svn:eol-style = native
Modified:
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/TraverseWithSortStrategy.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/TraverseWithSortStrategy.java?rev=1819936&r1=1819935&r2=1819936&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/TraverseWithSortStrategy.java
(original)
+++
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/TraverseWithSortStrategy.java
Wed Jan 3 09:48:29 2018
@@ -39,7 +39,6 @@ import javax.management.openmbean.Compos
import com.google.common.base.Stopwatch;
import org.apache.commons.io.FileUtils;
-import org.apache.jackrabbit.oak.commons.StringUtils;
import org.apache.jackrabbit.oak.commons.sort.ExternalSort;
import org.apache.jackrabbit.oak.index.indexer.document.NodeStateEntry;
import org.slf4j.Logger;
@@ -66,7 +65,7 @@ class TraverseWithSortStrategy implement
private final File storeDir;
private final boolean compressionEnabled;
private final Charset charset = UTF_8;
- private final Comparator<NodeStateEntryHolder> comparator;
+ private final Comparator<NodeStateHolder> comparator;
private NotificationEmitter emitter;
private MemoryListener listener;
private final int maxMemory =
Integer.getInteger(OAK_INDEXER_MAX_SORT_MEMORY_IN_GB,
OAK_INDEXER_MAX_SORT_MEMORY_IN_GB_DEFAULT);
@@ -78,7 +77,7 @@ class TraverseWithSortStrategy implement
private long memoryUsed;
private File sortWorkDir;
private List<File> sortedFiles = new ArrayList<>();
- private ArrayList<NodeStateEntryHolder> entryBatch = new ArrayList<>();
+ private ArrayList<NodeStateHolder> entryBatch = new ArrayList<>();
TraverseWithSortStrategy(Iterable<NodeStateEntry> nodeStates,
PathElementComparator pathComparator,
@@ -87,7 +86,7 @@ class TraverseWithSortStrategy implement
this.entryWriter = entryWriter;
this.storeDir = storeDir;
this.compressionEnabled = compressionEnabled;
- this.comparator = (e1, e2) -> pathComparator.compare(e1.pathElements,
e2.pathElements);
+ this.comparator = (e1, e2) ->
pathComparator.compare(e1.getPathElements(), e2.getPathElements());
}
@Override
@@ -105,11 +104,12 @@ class TraverseWithSortStrategy implement
}
private File sortStoreFile() throws IOException {
+ log.info("Proceeding to perform merge of {} sorted files",
sortedFiles.size());
Stopwatch w = Stopwatch.createStarted();
File sortedFile = new File(storeDir,
getSortedStoreFileName(compressionEnabled));
try(BufferedWriter writer = createWriter(sortedFile,
compressionEnabled)) {
- Function<String, NodeStateEntryHolder> func1 = (line) -> line ==
null ? null : new NodeStateEntryHolder(line);
- Function<NodeStateEntryHolder, String> func2 = holder -> holder ==
null ? null : holder.line;
+ Function<String, NodeStateHolder> func1 = (line) -> line == null ?
null : new SimpleNodeStateHolder(line);
+ Function<NodeStateHolder, String> func2 = holder -> holder == null
? null : holder.getLine();
ExternalSort.mergeSortedFiles(sortedFiles,
writer,
comparator,
@@ -152,7 +152,7 @@ class TraverseWithSortStrategy implement
String jsonText = entryWriter.asJson(e.getNodeState());
//Here logic differs from NodeStateEntrySorter in sense that
//Holder line consist only of json and not 'path|json'
- NodeStateEntryHolder h = new NodeStateEntryHolder(e.getPath(),
jsonText);
+ NodeStateHolder h = new StateInBytesHolder(e.getPath(), jsonText);
entryBatch.add(h);
updateMemoryUsed(h);
@@ -173,9 +173,9 @@ class TraverseWithSortStrategy implement
File newtmpfile = File.createTempFile("sortInBatch", "flatfile",
sortWorkDir);
long textSize = 0;
try (BufferedWriter writer =
FlatFileStoreUtils.createWriter(newtmpfile, compressionEnabled)) {
- for (NodeStateEntryHolder h : entryBatch) {
+ for (NodeStateHolder h : entryBatch) {
//Here holder line only contains nodeState json
- String text = entryWriter.toString(h.pathElements, h.line);
+ String text = entryWriter.toString(h.getPathElements(),
h.getLine());
writer.write(text);
writer.newLine();
textSize += text.length() + 1;
@@ -193,11 +193,8 @@ class TraverseWithSortStrategy implement
return !sufficientMemory.get();
}
- private void updateMemoryUsed(NodeStateEntryHolder h) {
- for (String e : h.pathElements) {
- memoryUsed += StringUtils.estimateMemoryUsage(e);
- }
- memoryUsed += StringUtils.estimateMemoryUsage(h.line);
+ private void updateMemoryUsed(NodeStateHolder h) {
+ memoryUsed += h.getMemorySize();
}
private static File createdSortWorkDir(File storeDir) throws IOException {
