This is an automated email from the ASF dual-hosted git repository.
stefanegli pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git
The following commit(s) were added to refs/heads/trunk by this push:
new 09e1ff4dd7 OAK-10347 : Adding small util class for exporting a tree as
flat file (#1021)
09e1ff4dd7 is described below
commit 09e1ff4dd795c1e7de45d5fa13aa209ac9713a2d
Author: stefan-egli <[email protected]>
AuthorDate: Mon Aug 7 13:45:51 2023 +0200
OAK-10347 : Adding small util class for exporting a tree as flat file
(#1021)
* OAK-10347 : Adding small util class for exporting a tree as flat file
<truncated/>
---------
Co-authored-by: Rishabh Kumar <[email protected]>
---
.../document/flatfile/NodeStateEntryWriter.java | 24 ++++-
.../document/flatfile/SimpleFlatFileUtil.java | 104 +++++++++++++++++++
.../document/flatfile/SimpleFlatFileUtilTest.java | 110 +++++++++++++++++++++
.../document/DocumentDiscoveryLiteServiceTest.java | 8 +-
4 files changed, 242 insertions(+), 4 deletions(-)
diff --git
a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/NodeStateEntryWriter.java
b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/NodeStateEntryWriter.java
index 690b995d94..4edd9fd923 100644
---
a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/NodeStateEntryWriter.java
+++
b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/NodeStateEntryWriter.java
@@ -30,7 +30,10 @@ import org.apache.jackrabbit.oak.spi.state.NodeState;
import java.io.IOException;
import java.io.Writer;
+import java.util.Comparator;
import java.util.List;
+import java.util.stream.Stream;
+import java.util.stream.StreamSupport;
import static org.apache.jackrabbit.guava.common.base.Preconditions.checkState;
@@ -40,13 +43,19 @@ public class NodeStateEntryWriter {
private final JsopBuilder jw = new JsopBuilder();
private final JsonSerializer serializer;
private final Joiner pathJoiner = Joiner.on('/');
+ private final boolean includeChildOrder;
//TODO Possible optimizations
//1. Compression
//2. Dictionary for properties
public NodeStateEntryWriter(BlobStore blobStore) {
+ this(blobStore, false);
+ }
+
+ public NodeStateEntryWriter(BlobStore blobStore, boolean
includeChildOrder) {
this.serializer = new JsonSerializer(jw, new
BlobIdSerializer(blobStore));
+ this.includeChildOrder = includeChildOrder;
}
public String toString(NodeStateEntry e) {
@@ -77,21 +86,30 @@ public class NodeStateEntryWriter {
}
public String asJson(NodeState nodeState) {
+ return
asJson(StreamSupport.stream(nodeState.getProperties().spliterator(), false));
+ }
+
+ String asSortedJson(NodeState nodeState) {
+ return
asJson(StreamSupport.stream(nodeState.getProperties().spliterator(), false)
+ .sorted(Comparator.comparing(PropertyState::getName)));
+ }
+
+ private String asJson(Stream<? extends PropertyState> stream) {
jw.resetWriter();
jw.object();
- for (PropertyState ps : nodeState.getProperties()) {
+ stream.forEach(ps -> {
String name = ps.getName();
if (include(name)) {
jw.key(name);
serializer.serialize(ps);
}
- }
+ });
jw.endObject();
return jw.toString();
}
private boolean include(String propertyName) {
- return !OAK_CHILD_ORDER.equals(propertyName);
+ return !OAK_CHILD_ORDER.equals(propertyName) || includeChildOrder;
}
//~-----------------------------------< Utilities to parse >
diff --git
a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/SimpleFlatFileUtil.java
b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/SimpleFlatFileUtil.java
new file mode 100644
index 0000000000..5c54bc6548
--- /dev/null
+++
b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/SimpleFlatFileUtil.java
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.jackrabbit.oak.index.indexer.document.flatfile;
+
+import static org.apache.jackrabbit.guava.common.collect.ImmutableList.copyOf;
+import static org.apache.jackrabbit.oak.commons.PathUtils.elements;
+
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.IOException;
+import java.io.Writer;
+import java.nio.file.Files;
+import java.util.Comparator;
+import java.util.stream.StreamSupport;
+
+import org.apache.jackrabbit.oak.index.indexer.document.NodeStateEntry;
+import org.apache.jackrabbit.oak.plugins.document.DocumentNodeState;
+import org.apache.jackrabbit.oak.spi.state.ChildNodeEntry;
+import org.apache.jackrabbit.oak.spi.state.NodeState;
+import org.apache.jackrabbit.oak.spi.state.NodeStateUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * This util class can be used to export a tree (eg entire repository) to a
flat
+ * file, without index dependency/involvement.
+ */
+public class SimpleFlatFileUtil {
+
+ private static final Logger log =
LoggerFactory.getLogger(SimpleFlatFileUtil.class);
+
+ private static final String LINE_SEPARATOR =
System.getProperty("line.separator");
+
+ private final Writer writer;
+ private final NodeStateEntryWriter entryWriter;
+ private long totalLines = 0;
+
+ private SimpleFlatFileUtil(Writer writer) {
+ // blobStore is only used for deserialization - so pass null here:
+ this.entryWriter = new NodeStateEntryWriter(null, true);
+ this.writer = writer;
+ }
+
+ public static void createFlatFileFor(NodeState ns, File f) throws
IOException {
+ log.info("createFlatFileFor : writing to {}", f.getCanonicalPath());
+ try (BufferedWriter bw = Files.newBufferedWriter(f.toPath())) {
+ createFlatFileFor(ns, bw);
+ }
+ }
+
+ public static void createFlatFileFor(NodeState ns, Writer writer) throws
IOException {
+ SimpleFlatFileUtil h = new SimpleFlatFileUtil(writer);
+ h.addEntryAndTraverseChildren(ns);
+ log.info("createFlatFileFor : done. wrote {} lines in total.",
h.totalLines);
+ }
+
+ private void addEntryAndTraverseChildren(NodeState ns) throws IOException {
+ addEntry(ns);
+ StreamSupport.stream(ns.getChildNodeEntries().spliterator(), false)
+
.sorted(Comparator.comparing(ChildNodeEntry::getName)).forEach(e -> {
+ try {
+ addEntryAndTraverseChildren(e.getNodeState());
+ } catch (IOException e1) {
+ // NOSONAR
+ throw new RuntimeException(e1);
+ }
+ });
+ }
+
+ private void addEntry(NodeState ns) throws IOException {
+ DocumentNodeState dns = (DocumentNodeState) ns;
+ NodeStateEntry e = new NodeStateEntry.NodeStateEntryBuilder(dns,
+ dns.getPath().toString()).build();
+ String path = e.getPath();
+ if (NodeStateUtils.isHiddenPath(path)) {
+ // skip
+ return;
+ }
+ String jsonText = entryWriter.asSortedJson(e.getNodeState());
+ String line = entryWriter.toString(copyOf(elements(path)), jsonText);
+ writer.append(line);
+ writer.append(LINE_SEPARATOR);
+ totalLines++;
+ if (totalLines % 10000 == 0) {
+ log.info("addEntry : wrote {} lines so far.", totalLines);
+ }
+ }
+}
diff --git
a/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/SimpleFlatFileUtilTest.java
b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/SimpleFlatFileUtilTest.java
new file mode 100644
index 0000000000..c13fc7cd33
--- /dev/null
+++
b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/SimpleFlatFileUtilTest.java
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.jackrabbit.oak.index.indexer.document.flatfile;
+
+import static org.apache.jackrabbit.oak.api.Type.NAMES;
+import static org.junit.Assert.assertEquals;
+
+import java.io.BufferedWriter;
+import java.io.StringWriter;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.jackrabbit.oak.plugins.document.DocumentMK;
+import org.apache.jackrabbit.oak.plugins.document.DocumentMKBuilderProvider;
+import org.apache.jackrabbit.oak.plugins.document.DocumentNodeStore;
+import org.apache.jackrabbit.oak.plugins.document.memory.MemoryDocumentStore;
+import org.apache.jackrabbit.oak.spi.commit.CommitInfo;
+import org.apache.jackrabbit.oak.spi.commit.EmptyHook;
+import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+
+public class SimpleFlatFileUtilTest {
+
+ private final String LINE_SEPARATOR = System.getProperty("line.separator");
+
+ @Rule
+ public final DocumentMKBuilderProvider builderProvider = new
DocumentMKBuilderProvider();
+
+ private DocumentNodeStore dns;
+
+ private StringWriter sw;
+
+ private BufferedWriter bw;
+
+ @Before
+ public void setup() throws Exception {
+ DocumentMK.Builder builder = builderProvider.newBuilder();
+ builder.setDocumentStore(new MemoryDocumentStore());
+ dns = builder.getNodeStore();
+ sw = new StringWriter();
+ bw = new BufferedWriter(sw);
+ }
+
+ @Test
+ public void testEmpty() throws Exception {
+ SimpleFlatFileUtil.createFlatFileFor(dns.getRoot(), bw);
+ bw.close();
+ assertEquals("/|{}" + LINE_SEPARATOR, sw.toString());
+ }
+
+ @Test
+ public void testSimple() throws Exception {
+ NodeBuilder b = dns.getRoot().builder();
+ b.child("child");
+ b.setProperty("prop", "value");
+ dns.merge(b, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+ SimpleFlatFileUtil.createFlatFileFor(dns.getRoot(), bw);
+ bw.close();
+ assertEquals(
+ "/|{\"prop\":\"value\"}" + LINE_SEPARATOR + "/child|{}" +
LINE_SEPARATOR,
+ sw.toString());
+ }
+
+ @Test
+ public void testPropertiesSorting() throws Exception {
+ NodeBuilder b = dns.getRoot().builder();
+ b.setProperty("g", "a");
+ b.setProperty("a", "z");
+ b.setProperty("c", "t");
+ dns.merge(b, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+ SimpleFlatFileUtil.createFlatFileFor(dns.getRoot(), bw);
+ bw.close();
+ assertEquals("/|{\"a\":\"z\",\"c\":\"t\",\"g\":\"a\"}" +
LINE_SEPARATOR,
+ sw.toString());
+ }
+
+ @Test
+ public void testChildOrder() throws Exception {
+ NodeBuilder b = dns.getRoot().builder();
+ b.setProperty("b", "2");
+ List<String> names = new ArrayList<>();
+ names.add("a");
+ names.add("x");
+ names.add("c");
+ b.setProperty(":childOrder", names, NAMES);
+ dns.merge(b, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+ SimpleFlatFileUtil.createFlatFileFor(dns.getRoot(), bw);
+ bw.close();
+
assertEquals("/|{\":childOrder\":[\"nam:a\",\"nam:x\",\"nam:c\"],\"b\":\"2\"}"
+ + LINE_SEPARATOR, sw.toString());
+ }
+}
diff --git
a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/DocumentDiscoveryLiteServiceTest.java
b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/DocumentDiscoveryLiteServiceTest.java
index 3b0ac7f43f..ed7cf9fd5c 100644
---
a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/DocumentDiscoveryLiteServiceTest.java
+++
b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/DocumentDiscoveryLiteServiceTest.java
@@ -23,6 +23,7 @@ import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
+import org.junit.Before;
import org.junit.Test;
import org.osgi.framework.BundleContext;
import org.osgi.service.component.ComponentContext;
@@ -34,6 +35,11 @@ import junitx.util.PrivateAccessor;
*/
public class DocumentDiscoveryLiteServiceTest extends
BaseDocumentDiscoveryLiteServiceTest {
+ @Before
+ public void setUp() {
+ ClusterNodeInfo.resetClockToDefault();
+ }
+
@Test
public void testActivateDeactivate() throws Exception {
// then test normal start with a DocumentNodeStore
@@ -115,7 +121,7 @@ public class DocumentDiscoveryLiteServiceTest extends
BaseDocumentDiscoveryLiteS
final ViewExpectation expectation1AfterShutdown = new
ViewExpectation(s1);
expectation1AfterShutdown.setActiveIds(s1.ns.getClusterId());
expectation1AfterShutdown.setInactiveIds(s2.ns.getClusterId());
- waitFor(expectation1AfterShutdown, 4000, "first should only see itself
after shutdown");
+ waitFor(expectation1AfterShutdown, 10000, "first should only see
itself after shutdown");
}
@Test
