Author: chetanm
Date: Fri Dec 8 07:25:28 2017
New Revision: 1817460
URL: http://svn.apache.org/viewvc?rev=1817460&view=rev
Log:
OAK-6353 - Use Document order traversal for reindexing performed on
DocumentNodeStore setups
Add support for writing and reading NodeStateEntries in json form
Added:
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/NodeStateEntryReader.java
(with props)
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/NodeStateEntryWriter.java
(with props)
jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/index/indexer/
jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/
jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/
jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/NodeStateEntryWriterTest.java
(with props)
Added:
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/NodeStateEntryReader.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/NodeStateEntryReader.java?rev=1817460&view=auto
==============================================================================
---
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/NodeStateEntryReader.java
(added)
+++
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/NodeStateEntryReader.java
Fri Dec 8 07:25:28 2017
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.jackrabbit.oak.index.indexer.document.flatfile;
+
+import org.apache.jackrabbit.oak.index.indexer.document.NodeStateEntry;
+import org.apache.jackrabbit.oak.json.BlobDeserializer;
+import org.apache.jackrabbit.oak.json.JsonDeserializer;
+import org.apache.jackrabbit.oak.plugins.blob.serializer.BlobIdSerializer;
+import org.apache.jackrabbit.oak.spi.blob.BlobStore;
+import org.apache.jackrabbit.oak.spi.state.NodeState;
+
+public class NodeStateEntryReader {
+ private final BlobDeserializer blobDeserializer;
+
+ public NodeStateEntryReader(BlobStore blobStore){
+ this.blobDeserializer = new BlobIdSerializer(blobStore);
+ }
+
+ public NodeStateEntry read(String line){
+ String[] parts = NodeStateEntryWriter.getParts(line);
+ return new NodeStateEntry(parseState(parts[1]), parts[0]);
+ }
+
+ private NodeState parseState(String part) {
+ JsonDeserializer des = new JsonDeserializer(blobDeserializer);
+ return des.deserialize(part);
+ }
+}
Propchange:
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/NodeStateEntryReader.java
------------------------------------------------------------------------------
svn:eol-style = native
Added:
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/NodeStateEntryWriter.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/NodeStateEntryWriter.java?rev=1817460&view=auto
==============================================================================
---
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/NodeStateEntryWriter.java
(added)
+++
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/NodeStateEntryWriter.java
Fri Dec 8 07:25:28 2017
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.jackrabbit.oak.index.indexer.document.flatfile;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.io.Writer;
+
+import org.apache.jackrabbit.oak.api.PropertyState;
+import org.apache.jackrabbit.oak.commons.json.JsopBuilder;
+import org.apache.jackrabbit.oak.index.indexer.document.NodeStateEntry;
+import org.apache.jackrabbit.oak.json.JsonSerializer;
+import org.apache.jackrabbit.oak.plugins.blob.serializer.BlobIdSerializer;
+import org.apache.jackrabbit.oak.spi.blob.BlobStore;
+import org.apache.jackrabbit.oak.spi.state.NodeState;
+
+import static com.google.common.base.Preconditions.checkState;
+import static com.google.common.base.StandardSystemProperty.LINE_SEPARATOR;
+
+public class NodeStateEntryWriter implements Closeable{
+ private static final String OAK_CHILD_ORDER = ":childOrder";
+ private static final String DELIMITER = "|";
+ private final Writer writer;
+ private final JsopBuilder jw = new JsopBuilder();
+ private final JsonSerializer serializer;
+
+ //TODO Possible optimizations
+ //1. Compression
+ //2. Dictionary for properties
+
+ public NodeStateEntryWriter(BlobStore blobStore, Writer writer) {
+ this.writer = writer;
+ this.serializer = new JsonSerializer(jw, new
BlobIdSerializer(blobStore));
+ }
+
+ public void write(NodeStateEntry e) throws IOException {
+ String text = asText(e.getNodeState());
+ writer.append(e.getPath())
+ .append(DELIMITER)
+ .append(text)
+ .append(LINE_SEPARATOR.value());
+ }
+
+ @Override
+ public void close() throws IOException {
+ writer.flush();
+ }
+
+ private String asText(NodeState nodeState) {
+ return asJson(nodeState);
+ }
+
+ private String asJson(NodeState nodeState) {
+ jw.resetWriter();
+ jw.object();
+ for (PropertyState ps : nodeState.getProperties()) {
+ String name = ps.getName();
+ if (include(name)) {
+ jw.key(name);
+ serializer.serialize(ps);
+ }
+ }
+ jw.endObject();
+ return jw.toString();
+ }
+
+ private boolean include(String propertyName) {
+ return !OAK_CHILD_ORDER.equals(propertyName);
+ }
+
+ //~-----------------------------------< Utilities to parse >
+
+ public static String getPath(String entryLine){
+ return entryLine.substring(0, getDelimiterPosition(entryLine));
+ }
+
+ public static String[] getParts(String line) {
+ int pos = getDelimiterPosition(line);
+ return new String[] {line.substring(0, pos), line.substring(pos + 1)};
+ }
+
+ private static int getDelimiterPosition(String entryLine) {
+ int indexOfPipe = entryLine.indexOf(NodeStateEntryWriter.DELIMITER);
+ checkState(indexOfPipe > 0, "Invalid path entry [%s]", entryLine);
+ return indexOfPipe;
+ }
+}
Propchange:
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/NodeStateEntryWriter.java
------------------------------------------------------------------------------
svn:eol-style = native
Added:
jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/NodeStateEntryWriterTest.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/NodeStateEntryWriterTest.java?rev=1817460&view=auto
==============================================================================
---
jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/NodeStateEntryWriterTest.java
(added)
+++
jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/NodeStateEntryWriterTest.java
Fri Dec 8 07:25:28 2017
@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.jackrabbit.oak.index.indexer.document.flatfile;
+
+import java.io.BufferedReader;
+import java.io.StringReader;
+import java.io.StringWriter;
+import java.util.Arrays;
+
+import org.apache.jackrabbit.oak.api.Type;
+import org.apache.jackrabbit.oak.index.indexer.document.NodeStateEntry;
+import org.apache.jackrabbit.oak.spi.blob.BlobStore;
+import org.apache.jackrabbit.oak.spi.blob.MemoryBlobStore;
+import org.apache.jackrabbit.oak.spi.state.EqualsDiff;
+import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
+import org.junit.Test;
+
+import static
org.apache.jackrabbit.oak.plugins.memory.EmptyNodeState.EMPTY_NODE;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+public class NodeStateEntryWriterTest {
+ private BlobStore blobStore = new MemoryBlobStore();
+ private NodeBuilder builder = EMPTY_NODE.builder();
+ private StringWriter sw = new StringWriter();
+
+ @Test
+ public void newLines() throws Exception{
+ NodeStateEntryWriter nw = new NodeStateEntryWriter(blobStore, sw);
+
+ builder.setProperty("foo", 1);
+ builder.setProperty("foo2", Arrays.asList("a", "b"), Type.STRINGS);
+ builder.setProperty("foo3", "text with \n new line");
+ nw.write(new NodeStateEntry(builder.getNodeState(), "/a"));
+ nw.close();
+
+ NodeStateEntryReader nr = new NodeStateEntryReader(blobStore);
+ BufferedReader br = new BufferedReader(new
StringReader(sw.toString()));
+
+ String line = br.readLine();
+ NodeStateEntry ne = nr.read(line);
+ assertEquals("/a", ne.getPath());
+ assertEquals("/a", NodeStateEntryWriter.getPath(line));
+ assertTrue(EqualsDiff.equals(ne.getNodeState(),
builder.getNodeState()));
+ }
+
+ @Test
+ public void multipleEntries() throws Exception{
+ NodeStateEntryWriter nw = new NodeStateEntryWriter(blobStore, sw);
+
+ NodeBuilder b1 = EMPTY_NODE.builder();
+ b1.setProperty("foo", "bar");
+
+ NodeBuilder b2 = EMPTY_NODE.builder();
+ b2.setProperty("foo2", "bar2");
+
+ NodeStateEntry e1 = new NodeStateEntry(b1.getNodeState(), "/a");
+ NodeStateEntry e2 = new NodeStateEntry(b2.getNodeState(), "/a");
+
+ nw.write(e1);
+ nw.write(e2);
+ nw.close();
+
+ NodeStateEntryReader nr = new NodeStateEntryReader(blobStore);
+ BufferedReader br = new BufferedReader(new
StringReader(sw.toString()));
+
+ assertEquals(e1, nr.read(br.readLine()));
+ assertEquals(e2, nr.read(br.readLine()));
+ }
+
+ @Test
+ public void childOrderNotWritten() throws Exception{
+ NodeStateEntryWriter nw = new NodeStateEntryWriter(blobStore, sw);
+
+ NodeBuilder b1 = EMPTY_NODE.builder();
+ b1.setProperty("foo", "bar");
+ b1.setProperty(":childOrder", "bar");
+ b1.setProperty(":hidden", "bar");
+
+ NodeStateEntry e1 = new NodeStateEntry(b1.getNodeState(), "/a");
+
+ nw.write(e1);
+ nw.close();
+
+ NodeStateEntryReader nr = new NodeStateEntryReader(blobStore);
+ BufferedReader br = new BufferedReader(new
StringReader(sw.toString()));
+
+ NodeStateEntry r1 = nr.read(br.readLine());
+ assertTrue(r1.getNodeState().hasProperty(":hidden"));
+ assertFalse(r1.getNodeState().hasProperty(":childOrder"));
+ }
+
+}
\ No newline at end of file
Propchange:
jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/NodeStateEntryWriterTest.java
------------------------------------------------------------------------------
svn:eol-style = native
