Author: chetanm Date: Fri Aug 11 10:22:56 2017 New Revision: 1804763 URL: http://svn.apache.org/viewvc?rev=1804763&view=rev Log: OAK-6545 - Tooling to serialize NodeState as json along with blobs
NodeStateSerializer supports serializing the NodeState values to json file with support for serializing blobs to FileDataStore Added: jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/exporter/ jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/exporter/FSBlobSerializer.java (with props) jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/exporter/JsopStreamWriter.java (with props) jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/exporter/NodeStateSerializer.java (with props) jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/exporter/ jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/exporter/FSBlobSerializerTest.java (with props) jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/exporter/NodeStateSerializerTest.java (with props) Added: jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/exporter/FSBlobSerializer.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/exporter/FSBlobSerializer.java?rev=1804763&view=auto ============================================================================== --- jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/exporter/FSBlobSerializer.java (added) +++ jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/exporter/FSBlobSerializer.java Fri Aug 11 10:22:56 2017 @@ -0,0 +1,91 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.jackrabbit.oak.exporter; + +import java.io.Closeable; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; + +import org.apache.jackrabbit.core.data.DataStoreException; +import org.apache.jackrabbit.core.data.FileDataStore; +import org.apache.jackrabbit.oak.api.Blob; +import org.apache.jackrabbit.oak.json.BlobDeserializer; +import org.apache.jackrabbit.oak.json.BlobSerializer; +import org.apache.jackrabbit.oak.plugins.blob.BlobStoreBlob; +import org.apache.jackrabbit.oak.plugins.blob.datastore.DataStoreBlobStore; +import org.apache.jackrabbit.oak.plugins.blob.datastore.OakFileDataStore; + +import static org.apache.commons.io.FileUtils.ONE_KB; + +/** + * Serializer which stores blobs in a FileDataStore format + */ +public class FSBlobSerializer extends BlobSerializer implements BlobDeserializer, Closeable { + private final File dir; + private final int maxInlineSize; + private final DataStoreBlobStore dataStore; + + public FSBlobSerializer(File dir) { + this(dir, (int) ONE_KB * 4); + } + + public FSBlobSerializer(File dir, int maxInlineSize) { + this.dir = dir; + this.maxInlineSize = maxInlineSize; + this.dataStore = createDataStore(); + } + + @Override + public String serialize(Blob blob) { + try { + try (InputStream is = blob.getNewStream()) { + return dataStore.writeBlob(is); + } + } catch (IOException e) { + throw new RuntimeException("Error occurred while serializing Blob " + + "with id " + blob.getContentIdentity(), e); + } + } + + private DataStoreBlobStore createDataStore() { + FileDataStore fds = new OakFileDataStore(); + fds.setPath(dir.getAbsolutePath()); + fds.setMinRecordLength(maxInlineSize); + fds.init(null); + return new DataStoreBlobStore(fds); + } + + @Override + public Blob deserialize(String value) { + return new BlobStoreBlob(dataStore, value); + } + + @Override + public void close() throws IOException { + if (dataStore != null) { + try { + dataStore.close(); + } catch (DataStoreException e) { + throw new IOException(e); + } + } + } +} Propchange: jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/exporter/FSBlobSerializer.java ------------------------------------------------------------------------------ svn:eol-style = native Added: jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/exporter/JsopStreamWriter.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/exporter/JsopStreamWriter.java?rev=1804763&view=auto ============================================================================== --- jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/exporter/JsopStreamWriter.java (added) +++ jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/exporter/JsopStreamWriter.java Fri Aug 11 10:22:56 2017 @@ -0,0 +1,160 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.jackrabbit.oak.exporter; + +import java.io.IOException; + +import com.google.gson.stream.JsonWriter; +import org.apache.jackrabbit.oak.commons.json.JsopWriter; + +/** + * A streaming JsopWriter which uses gson JsonWriter + */ +class JsopStreamWriter implements JsopWriter { + private final JsonWriter w; + + public JsopStreamWriter(JsonWriter w) { + this.w = w; + } + + @Override + public JsopWriter array() { + try { + w.beginArray(); + } catch (IOException e) { + throw new RuntimeException(e); + } + return this; + } + + @Override + public JsopWriter object() { + try { + w.beginObject(); + } catch (IOException e) { + throw new RuntimeException(e); + } + return this; + } + + @Override + public JsopWriter key(String key) { + try { + w.name(key); + } catch (IOException e) { + throw new RuntimeException(e); + } + return this; + } + + @Override + public JsopWriter value(String value) { + try { + w.value(value); + } catch (IOException e) { + throw new RuntimeException(e); + } + return this; + } + + @Override + public JsopWriter encodedValue(String raw) { + try { + //EncodedValue call is used in JsonSerializer for + //double values + try { + double d = Double.parseDouble(raw); + w.value(d); + } catch (NumberFormatException e) { + w.value(raw); + } + } catch (IOException e) { + throw new RuntimeException(e); + } + return this; + } + + @Override + public JsopWriter endObject() { + try { + w.endObject(); + } catch (IOException e) { + throw new RuntimeException(e); + } + return this; + } + + @Override + public JsopWriter endArray() { + try { + w.endArray(); + } catch (IOException e) { + throw new RuntimeException(e); + } + return this; + } + + @Override + public JsopWriter value(long x) { + try { + w.value(x); + } catch (IOException e) { + throw new RuntimeException(e); + } + return this; + } + + @Override + public JsopWriter value(boolean b) { + try { + w.value(b); + } catch (IOException e) { + throw new RuntimeException(e); + } + return this; + } + + //Unsupported operation. These are also not used by JsonSerializer + + @Override + public JsopWriter tag(char tag) { + throw new UnsupportedOperationException(); + } + + @Override + public JsopWriter append(JsopWriter diff) { + throw new UnsupportedOperationException(); + } + + @Override + public JsopWriter newline() { + throw new UnsupportedOperationException(); + } + + @Override + public void resetWriter() { + throw new UnsupportedOperationException(); + } + + @Override + public void setLineLength(int length) { + throw new UnsupportedOperationException(); + } +} Propchange: jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/exporter/JsopStreamWriter.java ------------------------------------------------------------------------------ svn:eol-style = native Added: jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/exporter/NodeStateSerializer.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/exporter/NodeStateSerializer.java?rev=1804763&view=auto ============================================================================== --- jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/exporter/NodeStateSerializer.java (added) +++ jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/exporter/NodeStateSerializer.java Fri Aug 11 10:22:56 2017 @@ -0,0 +1,174 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.jackrabbit.oak.exporter; + +import java.io.File; +import java.io.IOException; +import java.io.StringWriter; +import java.io.Writer; + +import com.google.common.io.Files; +import com.google.gson.stream.JsonWriter; +import org.apache.jackrabbit.oak.commons.json.JsopWriter; +import org.apache.jackrabbit.oak.json.Base64BlobSerializer; +import org.apache.jackrabbit.oak.json.BlobSerializer; +import org.apache.jackrabbit.oak.json.JsonSerializer; +import org.apache.jackrabbit.oak.spi.state.NodeState; +import org.apache.jackrabbit.oak.spi.state.NodeStateUtils; + +import static com.google.common.base.Charsets.UTF_8; +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkState; + +public class NodeStateSerializer { + public enum Format {JSON, TXT} + + private final NodeState nodeState; + private String blobDirName = "blobs"; + private String jsonFileName = "nodestates.json"; + private String txtFileName = "nodestates.txt"; + private int depth = Integer.MAX_VALUE; + private int maxChildNodes = Integer.MAX_VALUE; + private String filter = "{}"; + private File filterFile; + private String path = "/"; + private Format format = Format.JSON; + private boolean serializeBlobContent; + private boolean prettyPrint = true; + private FSBlobSerializer blobSerializer; + + public NodeStateSerializer(NodeState nodeState) { + this.nodeState = nodeState; + } + + public String serialize() throws IOException { + StringWriter sw = new StringWriter(); + serialize(sw, createBlobSerializer()); + closeSerializer(); + return sw.toString(); + } + + public void serialize(File dir) throws IOException { + if (dir.exists()) { + checkArgument(dir.isDirectory(), "Input file must be directory [%s]", dir.getAbsolutePath()); + } else { + checkState(dir.mkdirs(), "Cannot create directory [%s]", dir.getAbsolutePath()); + } + File file = new File(dir, getFileName()); + try (Writer writer = Files.newWriter(file, UTF_8)){ + serialize(writer, createBlobSerializer(dir)); + } + closeSerializer(); + } + + private void serialize(Writer writer, BlobSerializer blobSerializer) throws IOException { + JsopWriter jsopWriter = null; + if (format == Format.JSON) { + JsonWriter jw = new JsonWriter(writer); + if (prettyPrint) { + jw.setIndent(" "); + } + jsopWriter = new JsopStreamWriter(jw); + } + + serialize(jsopWriter, blobSerializer); + } + + private void serialize(JsopWriter writer, BlobSerializer blobSerializer) throws IOException { + JsonSerializer serializer = new JsonSerializer(writer, depth, 0, maxChildNodes, getFilter(), blobSerializer); + NodeState state = NodeStateUtils.getNode(nodeState, path); + serializer.serialize(state); + } + + private BlobSerializer createBlobSerializer(File dir) { + if (!serializeBlobContent) { + return new BlobSerializer(); + } + File blobs = new File(dir, blobDirName); + blobSerializer = new FSBlobSerializer(blobs); + return blobSerializer; + } + + private void closeSerializer() throws IOException { + if (blobSerializer != null) { + blobSerializer.close(); + } + } + + private BlobSerializer createBlobSerializer() { + return serializeBlobContent ? new Base64BlobSerializer() : new BlobSerializer(); + } + + private String getFilter() throws IOException { + return filterFile != null ? Files.toString(filterFile, UTF_8) : filter; + } + + public String getFileName() { + return format == Format.JSON ? jsonFileName : txtFileName; + } + + public String getBlobDirName() { + return blobDirName; + } + + public void setBlobDirName(String blobDirName) { + this.blobDirName = blobDirName; + } + + public void setJsonFileName(String jsonFileName) { + this.jsonFileName = jsonFileName; + } + + public void setTxtFileName(String txtFileName) { + this.txtFileName = txtFileName; + } + + public void setDepth(int depth) { + this.depth = depth; + } + + public void setMaxChildNodes(int maxChildNodes) { + this.maxChildNodes = maxChildNodes; + } + + public void setFilter(String filter) { + this.filter = filter; + } + + public void setFilterFile(File filterFile) { + this.filterFile = filterFile; + } + + public void setPath(String path) { + this.path = path; + } + + public void setFormat(Format format) { + this.format = format; + } + + public void setSerializeBlobContent(boolean serializeBlobContent) { + this.serializeBlobContent = serializeBlobContent; + } + + public void setPrettyPrint(boolean prettyPrint) { + this.prettyPrint = prettyPrint; + } +} Propchange: jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/exporter/NodeStateSerializer.java ------------------------------------------------------------------------------ svn:eol-style = native Added: jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/exporter/FSBlobSerializerTest.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/exporter/FSBlobSerializerTest.java?rev=1804763&view=auto ============================================================================== --- jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/exporter/FSBlobSerializerTest.java (added) +++ jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/exporter/FSBlobSerializerTest.java Fri Aug 11 10:22:56 2017 @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.jackrabbit.oak.exporter; + +import java.io.File; + +import joptsimple.internal.Strings; +import org.apache.jackrabbit.oak.api.Blob; +import org.apache.jackrabbit.oak.plugins.memory.AbstractBlob; +import org.apache.jackrabbit.oak.plugins.memory.ArrayBasedBlob; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import static com.google.common.base.Charsets.UTF_8; +import static org.junit.Assert.assertTrue; + +public class FSBlobSerializerTest { + + @Rule + public final TemporaryFolder folder = new TemporaryFolder(new File("target")); + + @Test + public void blobs() throws Exception{ + int maxInlineSize = 100; + FSBlobSerializer serializer = new FSBlobSerializer(folder.getRoot(), maxInlineSize); + String data = Strings.repeat('x', maxInlineSize * 10); + + Blob b = new ArrayBasedBlob(data.getBytes(UTF_8)); + + String id = serializer.serialize(b); + Blob b2 = serializer.deserialize(id); + + assertTrue(AbstractBlob.equal(b, b2)); + } + +} \ No newline at end of file Propchange: jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/exporter/FSBlobSerializerTest.java ------------------------------------------------------------------------------ svn:eol-style = native Added: jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/exporter/NodeStateSerializerTest.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/exporter/NodeStateSerializerTest.java?rev=1804763&view=auto ============================================================================== --- jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/exporter/NodeStateSerializerTest.java (added) +++ jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/exporter/NodeStateSerializerTest.java Fri Aug 11 10:22:56 2017 @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.jackrabbit.oak.exporter; + +import java.io.File; + +import com.google.common.io.Files; +import org.apache.jackrabbit.oak.json.BlobDeserializer; +import org.apache.jackrabbit.oak.json.JsonDeserializer; +import org.apache.jackrabbit.oak.spi.state.EqualsDiff; +import org.apache.jackrabbit.oak.spi.state.NodeBuilder; +import org.apache.jackrabbit.oak.spi.state.NodeState; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import static com.google.common.base.Charsets.UTF_8; +import static org.apache.jackrabbit.oak.plugins.memory.EmptyNodeState.EMPTY_NODE; +import static org.junit.Assert.*; +import static org.mockito.Mockito.mock; + +public class NodeStateSerializerTest { + private NodeBuilder builder = EMPTY_NODE.builder(); + private BlobDeserializer blobHandler = mock(BlobDeserializer.class); + + @Rule + public final TemporaryFolder folder = new TemporaryFolder(new File("target")); + + @Test + public void basics() throws Exception{ + builder.child("a").setProperty("foo", "bar"); + + NodeStateSerializer serializer = new NodeStateSerializer(builder.getNodeState()); + String json = serializer.serialize(); + NodeState nodeState2 = deserialize(json); + assertTrue(EqualsDiff.equals(builder.getNodeState(), nodeState2)); + } + + @Test + public void serializeToFile() throws Exception{ + builder.child("a").setProperty("foo", "bar"); + + NodeStateSerializer serializer = new NodeStateSerializer(builder.getNodeState()); + serializer.serialize(folder.getRoot()); + + File json = new File(folder.getRoot(), serializer.getFileName()); + assertTrue(json.exists()); + + String text = Files.toString(json, UTF_8); + NodeState nodeState2 = deserialize(text); + assertTrue(EqualsDiff.equals(builder.getNodeState(), nodeState2)); + + } + + private NodeState deserialize(String json) { + JsonDeserializer deserializer = new JsonDeserializer(blobHandler); + return deserializer.deserialize(json); + } + +} \ No newline at end of file Propchange: jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/exporter/NodeStateSerializerTest.java ------------------------------------------------------------------------------ svn:eol-style = native