Author: tomekr
Date: Wed Jul 27 12:24:42 2016
New Revision: 1754257
URL: http://svn.apache.org/viewvc?rev=1754257&view=rev
Log:
OAK-4500: Create tooling for reducing a document-based repository to a list of
paths
Added:
jackrabbit/oak/trunk/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/IndexCopier.java
jackrabbit/oak/trunk/oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/IncludeIndexTest.java
Modified:
jackrabbit/oak/trunk/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/RepositorySidegrade.java
jackrabbit/oak/trunk/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/MigrationFactory.java
jackrabbit/oak/trunk/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/parser/MigrationOptions.java
jackrabbit/oak/trunk/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/parser/OptionParserFactory.java
jackrabbit/oak/trunk/oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/cli/AbstractOak2OakTest.java
Added:
jackrabbit/oak/trunk/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/IndexCopier.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/IndexCopier.java?rev=1754257&view=auto
==============================================================================
---
jackrabbit/oak/trunk/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/IndexCopier.java
(added)
+++
jackrabbit/oak/trunk/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/IndexCopier.java
Wed Jul 27 12:24:42 2016
@@ -0,0 +1,136 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.jackrabbit.oak.upgrade;
+
+import org.apache.jackrabbit.oak.api.PropertyState;
+import org.apache.jackrabbit.oak.spi.state.ChildNodeEntry;
+import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
+import org.apache.jackrabbit.oak.spi.state.NodeState;
+import org.apache.jackrabbit.oak.upgrade.nodestate.NodeStateCopier;
+
+import java.util.Set;
+
+import static java.util.Collections.singleton;
+import static org.apache.commons.lang.StringUtils.isEmpty;
+import static
org.apache.jackrabbit.oak.plugins.index.IndexConstants.INDEX_CONTENT_NODE_NAME;
+import static
org.apache.jackrabbit.oak.plugins.index.IndexConstants.INDEX_DEFINITIONS_NAME;
+import static
org.apache.jackrabbit.oak.plugins.index.IndexConstants.TYPE_PROPERTY_NAME;
+import static
org.apache.jackrabbit.oak.plugins.index.IndexConstants.UNIQUE_PROPERTY_NAME;
+
+public final class IndexCopier {
+
+ private IndexCopier() {
+ }
+
+ /**
+ * Copy all index definition and data from sourceRoot to targetRoot. The
+ * indexing data is filtered to include only content related to the passed
+ * list of paths.
+ *
+ * @param sourceRoot the source
+ * @param targetRoot the target
+ * @param includes indexing data for these paths will be copied
+ */
+ public static void copy(NodeState sourceRoot, NodeBuilder targetRoot,
Set<String> includes) {
+ NodeState oakIndex = sourceRoot.getChildNode(INDEX_DEFINITIONS_NAME);
+ NodeBuilder targetOakIndex = copySingleNode(oakIndex, targetRoot,
INDEX_DEFINITIONS_NAME);
+
+ for (ChildNodeEntry child : oakIndex.getChildNodeEntries()) {
+ NodeState indexDef = child.getNodeState();
+ String type = indexDef.getString(TYPE_PROPERTY_NAME);
+ if (isEmpty(type)) {
+ continue;
+ }
+
+ NodeBuilder targetIndexDef = copySingleNode(child, targetOakIndex);
+
+ switch (type) {
+ case "property":
+ if (indexDef.getBoolean(UNIQUE_PROPERTY_NAME)) {
+ copyUniqueIndex(indexDef, targetIndexDef, includes);
+ } else {
+ copyMirrorIndex(indexDef, targetIndexDef, includes);
+ }
+ break;
+
+ case "counter":
+ copyMirrorIndex(indexDef, targetIndexDef, includes);
+
+ case "lucene":
+ copyLuceneIndex(indexDef, targetIndexDef, includes);
+
+ default:
+ break;
+ }
+ }
+ }
+
+ private static void copyUniqueIndex(NodeState indexDef, NodeBuilder
targetIndexDef, Set<String> includes) {
+ NodeState indexNode = indexDef.getChildNode(INDEX_CONTENT_NODE_NAME);
+ NodeBuilder targetIndexNode = copySingleNode(indexNode,
targetIndexDef, INDEX_CONTENT_NODE_NAME);
+
+ for (ChildNodeEntry attr : indexNode.getChildNodeEntries()) {
+ Iterable<String> entries = attr.getNodeState().getStrings("entry");
+ if (entries != null) {
+ for (String e : entries) {
+ if (startsWithAny(e, includes)) {
+ copySingleNode(attr, targetIndexNode);
+ }
+ }
+ }
+ }
+ }
+
+ private static void copyMirrorIndex(NodeState indexDef, NodeBuilder
targetIndexDef, Set<String> includes) {
+ NodeState indexNode = indexDef.getChildNode(INDEX_CONTENT_NODE_NAME);
+ NodeBuilder targetIndexNode = copySingleNode(indexNode,
targetIndexDef, INDEX_CONTENT_NODE_NAME);
+
+ for (ChildNodeEntry attr : indexNode.getChildNodeEntries()) {
+ NodeBuilder targetAttr = copySingleNode(attr, targetIndexNode);
+ NodeStateCopier.builder()
+ .include(includes)
+ .copy(attr.getNodeState(), targetAttr);
+ }
+ }
+
+ private static void copyLuceneIndex(NodeState indexDef, NodeBuilder
targetIndexDef, Set<String> includes) {
+ NodeStateCopier.builder()
+ .include(singleton("/"))
+ .copy(indexDef, targetIndexDef);
+ }
+
+ private static NodeBuilder copySingleNode(ChildNodeEntry source,
NodeBuilder targetParent) {
+ return copySingleNode(source.getNodeState(), targetParent,
source.getName());
+ }
+
+ private static NodeBuilder copySingleNode(NodeState source, NodeBuilder
targetParent, String name) {
+ NodeBuilder target = targetParent.child(name);
+ for (PropertyState p : source.getProperties()) {
+ target.setProperty(p);
+ }
+ return target;
+ }
+
+ private static boolean startsWithAny(String subject, Iterable<String>
patterns) {
+ for (String p : patterns) {
+ if (subject.startsWith(p)) {
+ return true;
+ }
+ }
+ return false;
+ }
+}
Modified:
jackrabbit/oak/trunk/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/RepositorySidegrade.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/RepositorySidegrade.java?rev=1754257&r1=1754256&r2=1754257&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/RepositorySidegrade.java
(original)
+++
jackrabbit/oak/trunk/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/RepositorySidegrade.java
Wed Jul 27 12:24:42 2016
@@ -91,6 +91,8 @@ public class RepositorySidegrade {
*/
private Set<String> mergePaths = DEFAULT_MERGE_PATHS;
+ private boolean includeIndex = false;
+
private boolean filterLongNames = true;
private boolean skipInitialization = false;
@@ -187,6 +189,9 @@ public class RepositorySidegrade {
this.excludePaths = copyOf(checkNotNull(excludes));
}
+ public void setIncludeIndex(boolean includeIndex) {
+ this.includeIndex = includeIndex;
+ }
/**
* Sets the paths that should be merged when the source repository
@@ -273,8 +278,19 @@ public class RepositorySidegrade {
private void copyState(NodeState sourceRoot, NodeBuilder targetRoot)
throws CommitFailedException {
copyWorkspace(sourceRoot, targetRoot);
+ if (includeIndex) {
+ IndexCopier.copy(sourceRoot, targetRoot, includePaths);
+ }
+
+ boolean isRemoveCheckpointReferences = false;
if (!copyCheckpoints(targetRoot)) {
LOG.info("Copying checkpoints is not supported for this
combination of node stores");
+ isRemoveCheckpointReferences = true;
+ }
+ if (!DEFAULT_INCLUDE_PATHS.equals(includePaths)) {
+ isRemoveCheckpointReferences = true;
+ }
+ if (isRemoveCheckpointReferences) {
removeCheckpointReferences(targetRoot);
}
Modified:
jackrabbit/oak/trunk/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/MigrationFactory.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/MigrationFactory.java?rev=1754257&r1=1754256&r2=1754257&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/MigrationFactory.java
(original)
+++
jackrabbit/oak/trunk/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/MigrationFactory.java
Wed Jul 27 12:24:42 2016
@@ -117,6 +117,7 @@ public class MigrationFactory {
}
sidegrade.setFilterLongNames(stores.getSrcType().isSupportLongNames()
&& !stores.getDstType().isSupportLongNames());
sidegrade.setSkipInitialization(options.isSkipInitialization());
+ sidegrade.setIncludeIndex(options.isIncludeIndex());
return sidegrade;
}
Modified:
jackrabbit/oak/trunk/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/parser/MigrationOptions.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/parser/MigrationOptions.java?rev=1754257&r1=1754256&r2=1754257&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/parser/MigrationOptions.java
(original)
+++
jackrabbit/oak/trunk/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/parser/MigrationOptions.java
Wed Jul 27 12:24:42 2016
@@ -46,6 +46,8 @@ public class MigrationOptions {
private final String[] mergePaths;
+ private final boolean includeIndex;
+
private final boolean failOnError;
private final boolean earlyShutdown;
@@ -78,6 +80,7 @@ public class MigrationOptions {
this.includePaths =
split(args.getOption(OptionParserFactory.INCLUDE_PATHS));
this.excludePaths =
split(args.getOption(OptionParserFactory.EXCLUDE_PATHS));
this.mergePaths =
split(args.getOption(OptionParserFactory.MERGE_PATHS));
+ this.includeIndex = args.hasOption(OptionParserFactory.INCLUDE_INDEX);
this.failOnError = args.hasOption(OptionParserFactory.FAIL_ON_ERROR);
this.earlyShutdown =
args.hasOption(OptionParserFactory.EARLY_SHUTDOWN);
this.skipInitialization =
args.hasOption(OptionParserFactory.SKIP_INIT);
@@ -133,6 +136,10 @@ public class MigrationOptions {
return skipNameCheck;
}
+ public boolean isIncludeIndex() {
+ return includeIndex;
+ }
+
private void logOptions() {
if (copyBinariesByReference) {
log.info("DataStore needs to be shared with new repository");
@@ -180,6 +187,10 @@ public class MigrationOptions {
log.info("Test for long-named nodes will be disabled");
}
+ if (includeIndex) {
+ log.info("Index data for the paths {} will be copied", (Object)
includePaths);
+ }
+
log.info("Cache size: {} MB", cacheSizeInMB);
}
Modified:
jackrabbit/oak/trunk/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/parser/OptionParserFactory.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/parser/OptionParserFactory.java?rev=1754257&r1=1754256&r2=1754257&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/parser/OptionParserFactory.java
(original)
+++
jackrabbit/oak/trunk/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/parser/OptionParserFactory.java
Wed Jul 27 12:24:42 2016
@@ -74,6 +74,8 @@ public class OptionParserFactory {
public static final String SKIP_NAME_CHECK = "skip-name-check";
+ public static final String INCLUDE_INDEX = "include-index";
+
public static OptionParser create() {
OptionParser op = new OptionParser();
addUsageOptions(op);
@@ -122,6 +124,7 @@ public class OptionParserFactory {
.ofType(String.class);
op.accepts(MERGE_PATHS, "Comma-separated list of paths to merge during
copy.").withRequiredArg()
.ofType(String.class);
+ op.accepts(INCLUDE_INDEX, "Copy index data for paths specified in the
" + INCLUDE_PATHS + " option");
}
private static void addVersioningOptions(OptionParser op) {
Added:
jackrabbit/oak/trunk/oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/IncludeIndexTest.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/IncludeIndexTest.java?rev=1754257&view=auto
==============================================================================
---
jackrabbit/oak/trunk/oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/IncludeIndexTest.java
(added)
+++
jackrabbit/oak/trunk/oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/IncludeIndexTest.java
Wed Jul 27 12:24:42 2016
@@ -0,0 +1,131 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.jackrabbit.oak.upgrade;
+
+import com.google.common.base.Joiner;
+import org.apache.jackrabbit.oak.commons.IOUtils;
+import org.apache.jackrabbit.oak.commons.PathUtils;
+import org.apache.jackrabbit.oak.spi.state.NodeState;
+import org.apache.jackrabbit.oak.spi.state.NodeStore;
+import org.apache.jackrabbit.oak.upgrade.cli.AbstractOak2OakTest;
+import org.apache.jackrabbit.oak.upgrade.cli.OakUpgrade;
+import org.apache.jackrabbit.oak.upgrade.cli.container.BlobStoreContainer;
+import org.apache.jackrabbit.oak.upgrade.cli.container.FileDataStoreContainer;
+import org.apache.jackrabbit.oak.upgrade.cli.container.MongoNodeStoreContainer;
+import org.apache.jackrabbit.oak.upgrade.cli.container.NodeStoreContainer;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.jcr.RepositoryException;
+import java.io.IOException;
+
+import static junit.framework.TestCase.assertTrue;
+import static
org.apache.jackrabbit.oak.upgrade.cli.container.MongoNodeStoreContainer.isMongoAvailable;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assume.assumeTrue;
+
+public class IncludeIndexTest extends AbstractOak2OakTest {
+
+ private static final Logger log =
LoggerFactory.getLogger(IncludeIndexTest.class);
+
+ private final BlobStoreContainer blob;
+
+ private final NodeStoreContainer source;
+
+ private final NodeStoreContainer destination;
+
+ private NodeStore nodeStore;
+
+ @Before
+ public void prepare() throws Exception {
+ NodeStore source = getSourceContainer().open();
+ try {
+ initContent(source);
+ } finally {
+ getSourceContainer().close();
+ }
+
+ String[] args = getArgs();
+ log.info("oak2oak {}", Joiner.on(' ').join(args));
+ OakUpgrade.main(args);
+
+ nodeStore = destination.open();
+ }
+
+ @After
+ public void clean() throws IOException {
+ IOUtils.closeQuietly(getDestinationContainer());
+ getDestinationContainer().clean();
+ getSourceContainer().clean();
+ }
+
+ public IncludeIndexTest() throws IOException {
+ assumeTrue(isMongoAvailable());
+ blob = new FileDataStoreContainer();
+ source = new MongoNodeStoreContainer(blob);
+ destination = new MongoNodeStoreContainer(blob);
+ }
+
+ @Override
+ protected NodeStoreContainer getSourceContainer() {
+ return source;
+ }
+
+ @Override
+ protected NodeStoreContainer getDestinationContainer() {
+ return destination;
+ }
+
+ @Override
+ protected String[] getArgs() {
+ return new String[] { "--src-datastore", blob.getDescription(),
"--copy-versions=false", "--skip-init", "--include-index",
"--include-paths=/apps,/libs", source.getDescription(),
destination.getDescription() };
+ }
+
+ @Test
+ public void validateMigration() throws RepositoryException, IOException {
+
assertNodeExists("/oak:index/nodetype/:index/nt%3Afile/libs/sling/xss/config.xml");
+ assertNodeExists("/oak:index/uuid/:index/" +
getUuid("/apps/repl/components/repl/repl.html/jcr:content"));
+
+ assertNodeMissing("/oak:index/nodetype/:index/nt%3Afile/sling.css");
+ assertNodeMissing("/oak:index/uuid/:index/" +
getUuid("/index.html/jcr:content"));
+ }
+
+ private String getUuid(String path) {
+ return getNode(path).getString("jcr:uuid");
+ }
+
+ private void assertNodeMissing(String path) {
+ assertFalse(getNode(path).exists());
+ }
+
+ private void assertNodeExists(String path) {
+ assertTrue(getNode(path).exists());
+ }
+
+ private NodeState getNode(String path) {
+ NodeState ns = nodeStore.getRoot();
+ for (String element : PathUtils.elements(path)) {
+ ns = ns.getChildNode(element);
+ }
+ return ns;
+ }
+}
\ No newline at end of file
Modified:
jackrabbit/oak/trunk/oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/cli/AbstractOak2OakTest.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/cli/AbstractOak2OakTest.java?rev=1754257&r1=1754256&r2=1754257&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/cli/AbstractOak2OakTest.java
(original)
+++
jackrabbit/oak/trunk/oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/cli/AbstractOak2OakTest.java
Wed Jul 27 12:24:42 2016
@@ -115,7 +115,7 @@ public abstract class AbstractOak2OakTes
}
}
- private void initContent(NodeStore target) throws IOException,
RepositoryException, CommitFailedException {
+ protected void initContent(NodeStore target) throws IOException,
RepositoryException, CommitFailedException {
NodeStore initialContent = testContent.open();
try {
RepositorySidegrade sidegrade = new
RepositorySidegrade(initialContent, target);
