Author: tomekr
Date: Tue Nov 21 10:03:13 2017
New Revision: 1815901
URL: http://svn.apache.org/viewvc?rev=1815901&view=rev
Log:
OAK-5933: Checkpoints are not sorted correctly in RepositorySidegrade
OAK-6003: Allow to migrate checkpoints for all type of sidegrades
OAK-6011: Test failure: JdbcToSegmentTest:validateMigration
OAK-6131: No need to rebuild the counter/uuid index anymore
OAK-6188: Allow to exclude nodes containing name fragment during the migration
OAK-6190: Allow to migrate checkpoints even if the custom include paths are
specified
OAK-6306: Upgrade uses lucene wrong version (transient dependency)
OAK-6336: Use the oak-blob-cloud for s3 support in oak-upgrade
OAK-6360: Extra diagnostic info for the checkpoint migration
OAK-6454: Inaccurate data in the oak-upgrade progress logger
OAK-6560: Sidegrade uses too much memory
OAK-6595: Allow to include indexing data for the included paths
OAK-6611: Many S3DataStore errors during migration with oak-upgrade
OAK-6633: Overwriting a versionable node with the copy-versions=false doesn't
remove versionable properties
OAK-6878: Populate S3DataStore fields with the passed properties
Added:
jackrabbit/oak/branches/1.2/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/plugins/segment/
jackrabbit/oak/branches/1.2/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/plugins/segment/CheckpointAccessor.java
- copied, changed from r1814391,
jackrabbit/oak/branches/1.2/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/node/TarNodeStore.java
jackrabbit/oak/branches/1.2/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/IndexCopier.java
jackrabbit/oak/branches/1.2/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/PersistingDiff.java
jackrabbit/oak/branches/1.2/oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/IncludeIndexTest.java
jackrabbit/oak/branches/1.2/oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/cli/blob/S3DataStoreFactoryTest.java
Removed:
jackrabbit/oak/branches/1.2/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/node/TarNodeStore.java
Modified:
jackrabbit/oak/branches/1.2/oak-upgrade/pom.xml
jackrabbit/oak/branches/1.2/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/RepositorySidegrade.java
jackrabbit/oak/branches/1.2/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/RepositoryUpgrade.java
jackrabbit/oak/branches/1.2/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/checkpoint/CheckpointRetriever.java
jackrabbit/oak/branches/1.2/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/MigrationFactory.java
jackrabbit/oak/branches/1.2/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/blob/S3DataStoreFactory.java
jackrabbit/oak/branches/1.2/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/node/SegmentFactory.java
jackrabbit/oak/branches/1.2/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/parser/MigrationCliArguments.java
jackrabbit/oak/branches/1.2/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/parser/MigrationOptions.java
jackrabbit/oak/branches/1.2/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/parser/OptionParserFactory.java
jackrabbit/oak/branches/1.2/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/parser/StoreArguments.java
jackrabbit/oak/branches/1.2/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/nodestate/AbstractDecoratedNodeState.java
jackrabbit/oak/branches/1.2/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/nodestate/FilteringNodeState.java
jackrabbit/oak/branches/1.2/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/nodestate/NameFilteringNodeState.java
jackrabbit/oak/branches/1.2/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/nodestate/NodeStateCopier.java
jackrabbit/oak/branches/1.2/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/nodestate/report/LoggingReporter.java
jackrabbit/oak/branches/1.2/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/nodestate/report/PeriodicReporter.java
jackrabbit/oak/branches/1.2/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/version/VersionCopier.java
jackrabbit/oak/branches/1.2/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/version/VersionHistoryUtil.java
jackrabbit/oak/branches/1.2/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/version/VersionableEditor.java
jackrabbit/oak/branches/1.2/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/version/VersionablePropertiesEditor.java
jackrabbit/oak/branches/1.2/oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/CopyVersionHistorySidegradeTest.java
jackrabbit/oak/branches/1.2/oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/CopyVersionHistoryTest.java
jackrabbit/oak/branches/1.2/oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/IncludeExcludeSidegradeTest.java
jackrabbit/oak/branches/1.2/oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/IncludeExcludeUpgradeTest.java
jackrabbit/oak/branches/1.2/oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/LongNameTest.java
jackrabbit/oak/branches/1.2/oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/cli/container/SegmentNodeStoreContainer.java
jackrabbit/oak/branches/1.2/oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/nodestate/FilteringNodeStateTest.java
Modified: jackrabbit/oak/branches/1.2/oak-upgrade/pom.xml
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.2/oak-upgrade/pom.xml?rev=1815901&r1=1815900&r2=1815901&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.2/oak-upgrade/pom.xml (original)
+++ jackrabbit/oak/branches/1.2/oak-upgrade/pom.xml Tue Nov 21 10:03:13 2017
@@ -68,6 +68,11 @@
<dependencies>
<dependency>
<groupId>org.apache.jackrabbit</groupId>
+ <artifactId>oak-blob-cloud</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.jackrabbit</groupId>
<artifactId>oak-core</artifactId>
<version>${project.version}</version>
</dependency>
@@ -83,13 +88,15 @@
</dependency>
<dependency>
<groupId>org.apache.jackrabbit</groupId>
- <artifactId>jackrabbit-aws-ext</artifactId>
+ <artifactId>jackrabbit-spi</artifactId>
<version>${jackrabbit.version}</version>
</dependency>
+
+ <!-- it has to match the version used in jackrabbit-core -->
<dependency>
- <groupId>org.apache.jackrabbit</groupId>
- <artifactId>jackrabbit-spi</artifactId>
- <version>${jackrabbit.version}</version>
+ <groupId>org.apache.lucene</groupId>
+ <artifactId>lucene-core</artifactId>
+ <version>3.6.0</version>
</dependency>
<dependency>
Copied:
jackrabbit/oak/branches/1.2/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/plugins/segment/CheckpointAccessor.java
(from r1814391,
jackrabbit/oak/branches/1.2/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/node/TarNodeStore.java)
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.2/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/plugins/segment/CheckpointAccessor.java?p2=jackrabbit/oak/branches/1.2/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/plugins/segment/CheckpointAccessor.java&p1=jackrabbit/oak/branches/1.2/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/node/TarNodeStore.java&r1=1814391&r2=1815901&rev=1815901&view=diff
==============================================================================
---
jackrabbit/oak/branches/1.2/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/node/TarNodeStore.java
(original)
+++
jackrabbit/oak/branches/1.2/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/plugins/segment/CheckpointAccessor.java
Tue Nov 21 10:03:13 2017
@@ -14,35 +14,17 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.jackrabbit.oak.upgrade.cli.node;
+package org.apache.jackrabbit.oak.plugins.segment;
import org.apache.jackrabbit.oak.spi.state.NodeState;
-import org.apache.jackrabbit.oak.spi.state.NodeStore;
-import org.apache.jackrabbit.oak.spi.state.ProxyNodeStore;
-public class TarNodeStore extends ProxyNodeStore {
+public final class CheckpointAccessor {
- private final NodeStore ns;
-
- private final SuperRootProvider superRootProvider;
-
- public TarNodeStore(NodeStore ns, SuperRootProvider superRootProvider) {
- this.ns = ns;
- this.superRootProvider = superRootProvider;
- }
-
- public NodeState getSuperRoot() {
- return superRootProvider.getSuperRoot();
+ private CheckpointAccessor() {
}
- @Override
- protected NodeStore getNodeStore() {
- return ns;
+ public static NodeState getCheckpointsRoot(SegmentNodeStore
segmentNodeStore) {
+ return segmentNodeStore.getCheckpoints();
}
- public interface SuperRootProvider {
-
- NodeState getSuperRoot();
-
- }
}
Added:
jackrabbit/oak/branches/1.2/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/IndexCopier.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.2/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/IndexCopier.java?rev=1815901&view=auto
==============================================================================
---
jackrabbit/oak/branches/1.2/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/IndexCopier.java
(added)
+++
jackrabbit/oak/branches/1.2/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/IndexCopier.java
Tue Nov 21 10:03:13 2017
@@ -0,0 +1,159 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.jackrabbit.oak.upgrade;
+
+import com.google.common.base.Predicate;
+import com.google.common.collect.Iterables;
+import org.apache.jackrabbit.oak.api.PropertyState;
+import org.apache.jackrabbit.oak.spi.state.ChildNodeEntry;
+import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
+import org.apache.jackrabbit.oak.spi.state.NodeState;
+import org.apache.jackrabbit.oak.upgrade.nodestate.NodeStateCopier;
+
+import javax.annotation.Nullable;
+import java.util.Set;
+
+import static java.util.Collections.singleton;
+import static org.apache.commons.lang.StringUtils.isEmpty;
+import static
org.apache.jackrabbit.oak.plugins.index.IndexConstants.INDEX_CONTENT_NODE_NAME;
+import static
org.apache.jackrabbit.oak.plugins.index.IndexConstants.INDEX_DEFINITIONS_NAME;
+import static
org.apache.jackrabbit.oak.plugins.index.IndexConstants.TYPE_PROPERTY_NAME;
+import static
org.apache.jackrabbit.oak.plugins.index.IndexConstants.UNIQUE_PROPERTY_NAME;
+
+public final class IndexCopier {
+
+ private IndexCopier() {
+ }
+
+ /**
+ * Copy all index definition and data from sourceRoot to targetRoot. The
+ * indexing data is filtered to include only content related to the passed
+ * list of paths.
+ *
+ * @param sourceRoot the source
+ * @param targetRoot the target
+ * @param includes indexing data for these paths will be copied
+ */
+ public static void copy(NodeState sourceRoot, NodeBuilder targetRoot,
Set<String> includes) {
+ NodeState oakIndex = sourceRoot.getChildNode(INDEX_DEFINITIONS_NAME);
+ NodeBuilder targetOakIndex = copySingleNode(oakIndex, targetRoot,
INDEX_DEFINITIONS_NAME);
+
+ for (ChildNodeEntry child : oakIndex.getChildNodeEntries()) {
+ NodeState indexDef = child.getNodeState();
+ String type = indexDef.getString(TYPE_PROPERTY_NAME);
+ if (isEmpty(type)) {
+ continue;
+ }
+
+ NodeBuilder targetIndexDef = copySingleNode(child, targetOakIndex);
+
+ if ("property".equals(type)) {
+ if (indexDef.getBoolean(UNIQUE_PROPERTY_NAME)) {
+ copyUniqueIndex(indexDef, targetIndexDef, includes);
+ } else {
+ copyMirrorIndex(indexDef, targetIndexDef, includes);
+ }
+ } else if ("counter".equals(type)) {
+ copyMirrorIndex(indexDef, targetIndexDef, includes);
+ } else if ("lucene".equals(type)) {
+ copyLuceneIndex(indexDef, targetIndexDef, includes);
+ }
+ }
+ }
+
+ private static void copyUniqueIndex(NodeState indexDef, NodeBuilder
targetIndexDef, Set<String> includes) {
+ for (ChildNodeEntry childIndexNode : getIndexNodes(indexDef)) {
+ NodeState indexNode = childIndexNode.getNodeState();
+ NodeBuilder targetIndexNode = copySingleNode(indexNode,
targetIndexDef, childIndexNode.getName());
+
+ boolean anyAttrCopied = false;
+ for (ChildNodeEntry attr : indexNode.getChildNodeEntries()) {
+ Iterable<String> entries =
attr.getNodeState().getStrings("entry");
+ if (entries != null) {
+ for (String e : entries) {
+ if (startsWithAny(e, includes)) {
+ copySingleNode(attr, targetIndexNode);
+ anyAttrCopied = true;
+ }
+ }
+ }
+ }
+ if (!anyAttrCopied) {
+ targetIndexNode.remove();
+ }
+ }
+ }
+
+ private static void copyMirrorIndex(NodeState indexDef, NodeBuilder
targetIndexDef, Set<String> includes) {
+ for (ChildNodeEntry childIndexNode : getIndexNodes(indexDef)) {
+ NodeState indexNode = childIndexNode.getNodeState();
+ NodeBuilder targetIndexNode = copySingleNode(indexNode,
targetIndexDef, childIndexNode.getName());
+
+ boolean anyAttrCopied = false;
+ for (ChildNodeEntry attr : indexNode.getChildNodeEntries()) {
+ NodeBuilder targetAttr = copySingleNode(attr, targetIndexNode);
+ boolean copied = NodeStateCopier.builder()
+ .include(includes)
+ .copy(attr.getNodeState(), targetAttr);
+ if (!copied) {
+ targetAttr.remove();
+ }
+ anyAttrCopied = copied || anyAttrCopied;
+ }
+ if (!anyAttrCopied) {
+ targetIndexNode.remove();
+ }
+ }
+ }
+
+ private static Iterable<? extends ChildNodeEntry> getIndexNodes(NodeState
indexDef) {
+ return Iterables.filter(indexDef.getChildNodeEntries(), new
Predicate<ChildNodeEntry>() {
+ @Override
+ public boolean apply(@Nullable ChildNodeEntry input) {
+ String name = input.getName();
+ return name.equals(INDEX_CONTENT_NODE_NAME) ||
name.startsWith(":oak:mount-");
+ }
+ });
+ }
+
+ private static void copyLuceneIndex(NodeState indexDef, NodeBuilder
targetIndexDef, Set<String> includes) {
+ NodeStateCopier.builder()
+ .include(singleton("/"))
+ .copy(indexDef, targetIndexDef);
+ }
+
+ private static NodeBuilder copySingleNode(ChildNodeEntry source,
NodeBuilder targetParent) {
+ return copySingleNode(source.getNodeState(), targetParent,
source.getName());
+ }
+
+ private static NodeBuilder copySingleNode(NodeState source, NodeBuilder
targetParent, String name) {
+ NodeBuilder target = targetParent.child(name);
+ for (PropertyState p : source.getProperties()) {
+ target.setProperty(p);
+ }
+ return target;
+ }
+
+ private static boolean startsWithAny(String subject, Iterable<String>
patterns) {
+ for (String p : patterns) {
+ if (subject.startsWith(p)) {
+ return true;
+ }
+ }
+ return false;
+ }
+}
Added:
jackrabbit/oak/branches/1.2/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/PersistingDiff.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.2/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/PersistingDiff.java?rev=1815901&view=auto
==============================================================================
---
jackrabbit/oak/branches/1.2/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/PersistingDiff.java
(added)
+++
jackrabbit/oak/branches/1.2/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/PersistingDiff.java
Tue Nov 21 10:03:13 2017
@@ -0,0 +1,179 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.jackrabbit.oak.upgrade;
+
+import org.apache.jackrabbit.oak.api.PropertyState;
+import org.apache.jackrabbit.oak.plugins.memory.MemoryNodeBuilder;
+import org.apache.jackrabbit.oak.plugins.segment.SegmentNodeState;
+import org.apache.jackrabbit.oak.plugins.segment.SegmentWriter;
+import org.apache.jackrabbit.oak.plugins.segment.file.FileStore;
+import org.apache.jackrabbit.oak.spi.blob.BlobStore;
+import org.apache.jackrabbit.oak.spi.state.NodeState;
+import org.apache.jackrabbit.oak.spi.state.NodeStateDiff;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.annotation.CheckForNull;
+import javax.annotation.Nonnull;
+import java.io.IOException;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+import static com.google.common.base.Preconditions.checkState;
+import static
org.apache.jackrabbit.oak.plugins.memory.EmptyNodeState.EMPTY_NODE;
+
+public class PersistingDiff implements NodeStateDiff {
+
+ private static final Logger LOG =
LoggerFactory.getLogger(PersistingDiff.class);
+
+ /**
+ * Number of content updates that need to happen before the updates
+ * are automatically purged to the underlying segments.
+ */
+ private static final int UPDATE_LIMIT =
+ Integer.getInteger("upgrade.update.limit", 10000);
+
+ private final SegmentWriter writer;
+
+ private final BlobStore blobStore;
+
+ private final PersistingDiff parent;
+
+ private final String nodeName;
+
+ @Nonnull
+ private MemoryNodeBuilder builder;
+
+ @Nonnull
+ private final NodeState base;
+
+ @CheckForNull
+ private IOException exception;
+
+ private long modCount;
+
+ private PersistingDiff(PersistingDiff parent, String nodeName, @Nonnull
NodeState base) {
+ this.writer = parent.writer;
+ this.blobStore = parent.blobStore;
+ this.builder = new MemoryNodeBuilder(checkNotNull(base));
+ this.parent = parent;
+ this.base = base;
+ this.nodeName = nodeName;
+ }
+
+ private PersistingDiff(SegmentWriter writer, BlobStore blobStore, @Nonnull
NodeState base) {
+ this.writer = writer;
+ this.blobStore = blobStore;
+ this.builder = new MemoryNodeBuilder(checkNotNull(base));
+ this.parent = null;
+ this.base = base;
+ this.nodeName = null;
+ }
+
+ public static SegmentNodeState applyDiffOnNodeState(
+ FileStore fileStore,
+ @Nonnull NodeState before,
+ @Nonnull NodeState after,
+ @Nonnull NodeState onto) throws IOException {
+ return new PersistingDiff(fileStore.getTracker().getWriter(),
fileStore.getBlobStore(), onto).diff(before, after);
+ }
+
+ private void updated() throws IOException {
+ if (modCount % UPDATE_LIMIT == 0) {
+ SegmentNodeState newBase =
writer.writeNode(builder.getNodeState());
+ builder = new MemoryNodeBuilder(newBase);
+ }
+ modCount++;
+ }
+
+ @CheckForNull
+ SegmentNodeState diff(@Nonnull NodeState before, @Nonnull NodeState after)
throws IOException {
+ boolean success = after.compareAgainstBaseState(before, this);
+ if (exception != null) {
+ throw new IOException(exception);
+ } else if (success) {
+ NodeState nodeState = builder.getNodeState();
+ checkState(modCount == 0 || !(nodeState instanceof
SegmentNodeState));
+ return writer.writeNode(nodeState);
+ } else {
+ return null;
+ }
+ }
+
+ @Override
+ public boolean propertyAdded(@Nonnull PropertyState after) {
+ builder.setProperty(after);
+ return true;
+ }
+
+ @Override
+ public boolean propertyChanged(@Nonnull PropertyState before, @Nonnull
PropertyState after) {
+ builder.setProperty(after);
+ return true;
+ }
+
+ @Override
+ public boolean propertyDeleted(PropertyState before) {
+ builder.removeProperty(before.getName());
+ return true;
+ }
+
+ @Override
+ public boolean childNodeAdded(@Nonnull String name, @Nonnull NodeState
after) {
+ try {
+ SegmentNodeState segmentNodeState = new PersistingDiff(this, name,
EMPTY_NODE).diff(EMPTY_NODE, after);
+ if (segmentNodeState != null) {
+ updated();
+ builder.setChildNode(name, segmentNodeState);
+ return true;
+ } else {
+ return false;
+ }
+ } catch (IOException e) {
+ exception = e;
+ return false;
+ }
+ }
+
+ @Override
+ public boolean childNodeChanged(@Nonnull String name, @Nonnull NodeState
before, @Nonnull NodeState after) {
+ try {
+ SegmentNodeState compacted = new PersistingDiff(this, name,
base.getChildNode(name)).diff(before, after);
+ if (compacted != null) {
+ updated();
+ builder.setChildNode(name, compacted);
+ return true;
+ } else {
+ return false;
+ }
+ } catch (IOException e) {
+ exception = e;
+ return false;
+ }
+ }
+
+ @Override
+ public boolean childNodeDeleted(String name, NodeState before) {
+ try {
+ updated();
+ builder.getChildNode(name).remove();
+ return true;
+ } catch (IOException e) {
+ exception = e;
+ return false;
+ }
+ }
+}
Modified:
jackrabbit/oak/branches/1.2/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/RepositorySidegrade.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.2/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/RepositorySidegrade.java?rev=1815901&r1=1815900&r2=1815901&view=diff
==============================================================================
---
jackrabbit/oak/branches/1.2/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/RepositorySidegrade.java
(original)
+++
jackrabbit/oak/branches/1.2/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/RepositorySidegrade.java
Tue Nov 21 10:03:13 2017
@@ -16,6 +16,7 @@
*/
package org.apache.jackrabbit.oak.upgrade;
+import java.io.IOException;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.LinkedHashMap;
@@ -28,9 +29,14 @@ import javax.jcr.RepositoryException;
import com.google.common.collect.Lists;
import org.apache.commons.lang.StringUtils;
+import org.apache.jackrabbit.JcrConstants;
import org.apache.jackrabbit.oak.api.CommitFailedException;
import org.apache.jackrabbit.oak.api.PropertyState;
import org.apache.jackrabbit.oak.api.Type;
+import org.apache.jackrabbit.oak.commons.PathUtils;
+import org.apache.jackrabbit.oak.plugins.nodetype.TypePredicate;
+import org.apache.jackrabbit.oak.plugins.segment.SegmentNodeState;
+import org.apache.jackrabbit.oak.plugins.segment.file.FileStore;
import org.apache.jackrabbit.oak.spi.commit.CommitHook;
import org.apache.jackrabbit.oak.spi.commit.CommitInfo;
import org.apache.jackrabbit.oak.spi.commit.CompositeEditorProvider;
@@ -43,11 +49,12 @@ import org.apache.jackrabbit.oak.spi.sta
import org.apache.jackrabbit.oak.spi.state.NodeStore;
import
org.apache.jackrabbit.oak.upgrade.RepositoryUpgrade.LoggingCompositeHook;
import org.apache.jackrabbit.oak.upgrade.checkpoint.CheckpointRetriever;
-import org.apache.jackrabbit.oak.upgrade.cli.node.TarNodeStore;
+import org.apache.jackrabbit.oak.upgrade.cli.node.SegmentFactory;
+import org.apache.jackrabbit.oak.upgrade.nodestate.FilteringNodeState;
import org.apache.jackrabbit.oak.upgrade.nodestate.NameFilteringNodeState;
+import org.apache.jackrabbit.oak.upgrade.nodestate.NodeStateCopier;
import org.apache.jackrabbit.oak.upgrade.nodestate.report.LoggingReporter;
import org.apache.jackrabbit.oak.upgrade.nodestate.report.ReportingNodeState;
-import org.apache.jackrabbit.oak.upgrade.nodestate.NodeStateCopier;
import org.apache.jackrabbit.oak.upgrade.version.VersionCopyConfiguration;
import org.apache.jackrabbit.oak.upgrade.version.VersionHistoryUtil;
import org.apache.jackrabbit.oak.upgrade.version.VersionableEditor;
@@ -62,7 +69,9 @@ import static org.apache.jackrabbit.JcrC
import static org.apache.jackrabbit.JcrConstants.JCR_SYSTEM;
import static
org.apache.jackrabbit.oak.spi.security.authorization.permission.PermissionConstants.NT_REP_PERMISSION_STORE;
import static
org.apache.jackrabbit.oak.spi.security.authorization.permission.PermissionConstants.REP_PERMISSION_STORE;
+import static
org.apache.jackrabbit.oak.upgrade.RepositoryUpgrade.DEFAULT_EXCLUDE_FRAGMENTS;
import static
org.apache.jackrabbit.oak.upgrade.RepositoryUpgrade.DEFAULT_EXCLUDE_PATHS;
+import static
org.apache.jackrabbit.oak.upgrade.RepositoryUpgrade.DEFAULT_FRAGMENT_PATHS;
import static
org.apache.jackrabbit.oak.upgrade.RepositoryUpgrade.DEFAULT_INCLUDE_PATHS;
import static
org.apache.jackrabbit.oak.upgrade.RepositoryUpgrade.DEFAULT_MERGE_PATHS;
import static
org.apache.jackrabbit.oak.upgrade.RepositoryUpgrade.calculateEffectiveIncludePaths;
@@ -76,7 +85,7 @@ public class RepositorySidegrade {
private static final Logger LOG =
LoggerFactory.getLogger(RepositorySidegrade.class);
- private static final int LOG_NODE_COPY =
Integer.getInteger("oak.upgrade.logNodeCopy", 10000);
+ static final int LOG_NODE_COPY =
Integer.getInteger("oak.upgrade.logNodeCopy", 10000);
private static final String WORKSPACE_NAME_PROP =
"oak.upgrade.workspaceName";
@@ -85,6 +94,8 @@ public class RepositorySidegrade {
*/
private final NodeStore target;
+ private final FileStore targetFileStore;
+
private final NodeStore source;
/**
@@ -98,12 +109,26 @@ public class RepositorySidegrade {
private Set<String> excludePaths = DEFAULT_EXCLUDE_PATHS;
/**
+ * Paths supporting fragments during the copy process. Empty by default.
+ */
+ private Set<String> fragmentPaths = DEFAULT_FRAGMENT_PATHS;
+
+ /**
+ * Fragments to exclude during the copy process. Empty by default.
+ */
+ private Set<String> excludeFragments = DEFAULT_EXCLUDE_FRAGMENTS;
+
+ /**
* Paths to merge during the copy process. Empty by default.
*/
private Set<String> mergePaths = DEFAULT_MERGE_PATHS;
private boolean skipCheckpoints = false;
+ private boolean forceCheckpoints = false;
+
+ private boolean includeIndex = false;
+
private boolean filterLongNames = true;
private boolean verify = false;
@@ -160,6 +185,12 @@ public class RepositorySidegrade {
public RepositorySidegrade(NodeStore source, NodeStore target) {
this.source = source;
this.target = target;
+
+ FileStore fs = null;
+ if (target instanceof SegmentFactory.NodeStoreWithFileStore) {
+ fs = ((SegmentFactory.NodeStoreWithFileStore)
target).getFileStore();
+ }
+ this.targetFileStore = fs;
}
/**
@@ -203,6 +234,29 @@ public class RepositorySidegrade {
}
/**
+ * Sets the paths that should support the fragments.
+ *
+ * @param fragmentPaths Paths that should support fragments.
+ */
+ public void setFragmentPaths(@Nonnull String... fragmentPaths) {
+ this.fragmentPaths = copyOf(checkNotNull(fragmentPaths));
+ }
+
+ /**
+ * Sets the name fragments that should be excluded when the source
repository
+ * is copied to the target repository.
+ *
+ * @param excludes Name fragments to be excluded from the copy.
+ */
+ public void setExcludeFragments(@Nonnull String... excludes) {
+ this.excludeFragments = copyOf(checkNotNull(excludes));
+ }
+
+ public void setIncludeIndex(boolean includeIndex) {
+ this.includeIndex = includeIndex;
+ }
+
+ /**
* Sets the paths that should be merged when the source repository
* is copied to the target repository.
*
@@ -228,6 +282,10 @@ public class RepositorySidegrade {
this.skipCheckpoints = skipCheckpoints;
}
+ public void setForceCheckpoints(boolean forceCheckpoints) {
+ this.forceCheckpoints = forceCheckpoints;
+ }
+
/**
* Copies the full content from the source to the target repository.
* <p>
@@ -264,14 +322,9 @@ public class RepositorySidegrade {
builder.setChildNode(":async");
}
- private void copyState() throws CommitFailedException, RepositoryException
{
- final List<CommitHook> hooks = new ArrayList<CommitHook>();
- if (customCommitHooks != null) {
- hooks.addAll(customCommitHooks);
- }
-
+ private void copyState() throws CommitFailedException,
RepositoryException, IOException {
boolean migrateCheckpoints = true;
- if (!isCompleteMigration()) {
+ if (!isCompleteMigration() && !forceCheckpoints) {
LOG.info("Checkpoints won't be migrated because of the specified
paths");
migrateCheckpoints = false;
}
@@ -283,35 +336,19 @@ public class RepositorySidegrade {
LOG.info("Checkpoints won't be migrated because of the
--skip-checkpoints options");
migrateCheckpoints = false;
}
+ if (targetExists()) {
+ LOG.info("Checkpoints won't be migrated because the destination
repository exists");
+ migrateCheckpoints = false;
+ }
if (migrateCheckpoints) {
migrateCheckpoints = migrateWithCheckpoints();
}
if (!migrateCheckpoints) {
- NodeState sourceRoot = wrapSource(source.getRoot());
- NodeBuilder targetRoot = target.getRoot().builder();
- copyWorkspace(sourceRoot, targetRoot);
- removeCheckpointReferences(targetRoot);
- if (!versionCopyConfiguration.isCopyAll()) {
- NodeBuilder versionStorage =
VersionHistoryUtil.getVersionStorage(targetRoot);
- if (!versionStorage.exists()) { // it's possible that this is
a new repository and the version storage
- // hasn't been created/copied
yet
- versionStorage =
VersionHistoryUtil.createVersionStorage(targetRoot);
- }
- if (!versionCopyConfiguration.skipOrphanedVersionsCopy()) {
- copyVersionStorage(targetRoot,
getVersionStorage(sourceRoot), versionStorage, versionCopyConfiguration);
- }
- hooks.add(new EditorHook(new
VersionableEditor.Provider(sourceRoot, getWorkspaceName(),
versionCopyConfiguration)));
- }
- // type validation, reference and indexing hooks
- hooks.add(new EditorHook(new CompositeEditorProvider(
- createTypeEditorProvider(),
- createIndexEditorProvider()
- )));
- target.merge(targetRoot, new LoggingCompositeHook(hooks, null,
false), CommitInfo.EMPTY);
+ migrateWithoutCheckpoints();
}
}
- private boolean migrateWithCheckpoints() throws CommitFailedException {
+ private boolean migrateWithCheckpoints() throws CommitFailedException,
IOException {
List<CheckpointRetriever.Checkpoint> checkpoints =
CheckpointRetriever.getCheckpoints(source);
if (checkpoints == null) {
return false;
@@ -319,25 +356,27 @@ public class RepositorySidegrade {
Map<String, String> nameToRevision = new LinkedHashMap<String,
String>();
Map<String, String> checkpointSegmentToDoc = new LinkedHashMap<String,
String>();
- NodeState previousRoot = null;
- NodeBuilder targetRoot = target.getRoot().builder();
+
+ NodeState initialRoot = target.getRoot();
+ NodeState targetRoot = initialRoot;
+ NodeState previousRoot = initialRoot;
for (CheckpointRetriever.Checkpoint checkpoint : checkpoints) {
NodeState checkpointRoot = source.retrieve(checkpoint.getName());
- if (previousRoot == null) {
+ Map<String, String> checkpointInfo =
source.checkpointInfo(checkpoint.getName());
+
+ boolean tracePaths;
+ if (previousRoot == initialRoot) {
LOG.info("Migrating first checkpoint: {}",
checkpoint.getName());
- NodeStateCopier.builder()
-
.include(calculateEffectiveIncludePaths(DEFAULT_INCLUDE_PATHS, checkpointRoot))
- .merge(of("/jcr:system"))
- .copy(wrapSource(checkpointRoot), targetRoot);
- copyProperties(checkpointRoot, targetRoot);
+ tracePaths = true;
} else {
LOG.info("Applying diff to {}", checkpoint.getName());
- checkpointRoot.compareAgainstBaseState(previousRoot, new
ApplyDiff(targetRoot));
+ tracePaths = false;
}
- target.merge(targetRoot, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+ LOG.info("Checkpoint expiry time: {}, metadata: {}",
checkpoint.getExpiryTime(), checkpointInfo);
+
+ targetRoot = copyDiffToTarget(previousRoot, checkpointRoot,
targetRoot, tracePaths);
previousRoot = checkpointRoot;
- Map<String, String> checkpointInfo =
source.checkpointInfo(checkpoint.getName());
String newCheckpointName =
target.checkpoint(checkpoint.getExpiryTime() - System.currentTimeMillis(),
checkpointInfo);
if (checkpointInfo.containsKey("name")) {
nameToRevision.put(checkpointInfo.get("name"),
newCheckpointName);
@@ -346,20 +385,20 @@ public class RepositorySidegrade {
}
NodeState sourceRoot = source.getRoot();
- if (previousRoot == null) {
+ boolean tracePaths;
+ if (previousRoot == initialRoot) {
LOG.info("No checkpoints found; migrating head");
- NodeStateCopier.builder()
-
.include(calculateEffectiveIncludePaths(DEFAULT_INCLUDE_PATHS, sourceRoot))
- .merge(of("/jcr:system"))
- .copy(wrapSource(sourceRoot), targetRoot);
- copyProperties(sourceRoot, targetRoot);
+ tracePaths = true;
} else {
LOG.info("Applying diff to head");
- sourceRoot.compareAgainstBaseState(previousRoot, new
ApplyDiff(targetRoot));
+ tracePaths = false;
}
+ targetRoot = copyDiffToTarget(previousRoot, sourceRoot, targetRoot,
tracePaths);
+
LOG.info("Rewriting checkpoint names in /:async {}", nameToRevision);
- NodeBuilder async = targetRoot.getChildNode(":async");
+ NodeBuilder targetBuilder = targetRoot.builder();
+ NodeBuilder async = targetBuilder.getChildNode(":async");
for (Map.Entry<String, String> e : nameToRevision.entrySet()) {
async.setProperty(e.getKey(), e.getValue(), Type.STRING);
@@ -375,13 +414,74 @@ public class RepositorySidegrade {
}
async.setProperty(e.getKey() + "-temp", tempValues, Type.STRINGS);
}
-
- target.merge(targetRoot, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+ target.merge(targetBuilder, EmptyHook.INSTANCE, CommitInfo.EMPTY);
return true;
}
+ private NodeState copyDiffToTarget(NodeState before, NodeState after,
NodeState targetRoot, boolean tracePaths) throws IOException,
CommitFailedException {
+
+ NodeBuilder targetBuilder = targetRoot.builder();
+ if (targetFileStore == null) {
+ NodeState currentRoot = wrapNodeState(after, tracePaths, true);
+ NodeState baseRoot = wrapNodeState(before, false, true);
+
+ currentRoot.compareAgainstBaseState(baseRoot, new
ApplyDiff(targetBuilder));
+ } else {
+ NodeState currentRoot = wrapNodeState(after, false, true);
+ NodeState baseRoot = wrapNodeState(before, false, true);
+
+ SegmentNodeState state =
PersistingDiff.applyDiffOnNodeState(targetFileStore, baseRoot, currentRoot,
targetRoot);
+ state.compareAgainstBaseState(targetRoot, new
ApplyDiff(targetBuilder));
+ }
+ return target.merge(targetBuilder, EmptyHook.INSTANCE,
CommitInfo.EMPTY);
+ }
+
+ private void migrateWithoutCheckpoints() throws CommitFailedException,
RepositoryException {
+ final List<CommitHook> hooks = new ArrayList<CommitHook>();
+ if (customCommitHooks != null) {
+ hooks.addAll(customCommitHooks);
+ }
+
+ NodeState sourceRoot = wrapNodeState(source.getRoot(), true, false);
+ NodeBuilder targetRoot = target.getRoot().builder();
+ copyWorkspace(sourceRoot, targetRoot);
+ removeCheckpointReferences(targetRoot);
+ if (includeIndex) {
+ IndexCopier.copy(sourceRoot, targetRoot, includePaths);
+ }
+ if (!versionCopyConfiguration.isCopyAll()) {
+ NodeBuilder versionStorage =
VersionHistoryUtil.getVersionStorage(targetRoot);
+ if (!versionStorage.exists()) { // it's possible that this is a
new repository and the version storage
+ // hasn't been created/copied yet
+ versionStorage =
VersionHistoryUtil.createVersionStorage(targetRoot);
+ }
+ if (!versionCopyConfiguration.skipOrphanedVersionsCopy()) {
+ copyVersionStorage(targetRoot, getVersionStorage(sourceRoot),
versionStorage, versionCopyConfiguration);
+ }
+ hooks.add(new EditorHook(new
VersionableEditor.Provider(sourceRoot, getWorkspaceName(),
versionCopyConfiguration)));
+ }
+ // type validation, reference and indexing hooks
+ hooks.add(new EditorHook(new CompositeEditorProvider(
+ createTypeEditorProvider(),
+ createIndexEditorProvider()
+ )));
+ target.merge(targetRoot, new LoggingCompositeHook(hooks, null, false),
CommitInfo.EMPTY);
+ removeVersions();
+ }
+
+ private void removeVersions() throws CommitFailedException {
+ NodeState root = target.getRoot();
+ NodeState wrappedRoot = FilteringNodeState.wrap("/", root,
includePaths, excludePaths, fragmentPaths, excludeFragments);
+ List<String> versionablesToStrip =
VersionHistoryUtil.getVersionableNodes(wrappedRoot, new TypePredicate(root,
JcrConstants.MIX_VERSIONABLE), versionCopyConfiguration.getVersionsMinDate());
+ if (!versionablesToStrip.isEmpty()) {
+ LOG.info("Removing version histories for included paths");
+ NodeBuilder newRoot = VersionHistoryUtil.removeVersions(root,
versionablesToStrip);
+ target.merge(newRoot, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+ }
+ }
+
private boolean isCompleteMigration() {
- return includePaths.equals(DEFAULT_INCLUDE_PATHS) &&
excludePaths.equals(DEFAULT_EXCLUDE_PATHS) &&
mergePaths.equals(DEFAULT_MERGE_PATHS);
+ return includePaths.equals(DEFAULT_INCLUDE_PATHS) &&
excludePaths.equals(DEFAULT_EXCLUDE_PATHS) &&
excludeFragments.equals(DEFAULT_EXCLUDE_FRAGMENTS) &&
mergePaths.equals(DEFAULT_MERGE_PATHS) &&
fragmentPaths.equals(DEFAULT_FRAGMENT_PATHS);
}
private void copyWorkspace(NodeState sourceRoot, NodeBuilder targetRoot) {
@@ -397,6 +497,8 @@ public class RepositorySidegrade {
NodeStateCopier.builder()
.include(includes)
.exclude(excludes)
+ .supportFragment(fragmentPaths)
+ .excludeFragments(excludeFragments)
.merge(merges)
.copy(sourceRoot, targetRoot);
@@ -441,16 +543,8 @@ public class RepositorySidegrade {
}
private void verify() {
- final NodeState sourceRoot;
- final NodeState targetRoot;
-
- if (source instanceof TarNodeStore && target instanceof TarNodeStore) {
- sourceRoot = ((TarNodeStore) source).getSuperRoot();
- targetRoot = ((TarNodeStore) target).getSuperRoot();
- } else {
- sourceRoot = source.getRoot();
- targetRoot = target.getRoot();
- }
+ final NodeState sourceRoot = source.getRoot();
+ final NodeState targetRoot = target.getRoot();
final NodeState reportingSource = ReportingNodeState.wrap(sourceRoot,
new LoggingReporter(LOG, "Verifying", LOG_NODE_COPY, -1));
@@ -462,14 +556,22 @@ public class RepositorySidegrade {
}
}
- private NodeState wrapSource(NodeState source) {
- final NodeState reportingSourceRoot = ReportingNodeState.wrap(source,
new LoggingReporter(LOG, "Copying", LOG_NODE_COPY, -1));
- final NodeState sourceRoot;
+ private NodeState wrapNodeState(NodeState source, boolean tracePaths,
boolean filterPaths) {
+ NodeState wrapped = source;
+ if (!isCompleteMigration() && filterPaths) {
+ wrapped = FilteringNodeState.wrap("/", wrapped, includePaths,
excludePaths, fragmentPaths, excludeFragments);
+ }
+ if (tracePaths) {
+ wrapped = ReportingNodeState.wrap(wrapped, new
LoggingReporter(LOG, "Copying", LOG_NODE_COPY, -1));
+ }
if (filterLongNames) {
- sourceRoot = NameFilteringNodeState.wrap(reportingSourceRoot);
- } else {
- sourceRoot = reportingSourceRoot;
+ wrapped = NameFilteringNodeState.wrap(wrapped);
}
- return sourceRoot;
+ return wrapped;
+ }
+
+ private boolean targetExists() {
+ return target.getRoot().getChildNodeEntries().iterator().hasNext();
}
+
}
Modified:
jackrabbit/oak/branches/1.2/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/RepositoryUpgrade.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.2/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/RepositoryUpgrade.java?rev=1815901&r1=1815900&r2=1815901&view=diff
==============================================================================
---
jackrabbit/oak/branches/1.2/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/RepositoryUpgrade.java
(original)
+++
jackrabbit/oak/branches/1.2/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/RepositoryUpgrade.java
Tue Nov 21 10:03:13 2017
@@ -30,9 +30,6 @@ import static org.apache.jackrabbit.oak.
import static
org.apache.jackrabbit.oak.plugins.nodetype.NodeTypeConstants.NODE_TYPES_PATH;
import static
org.apache.jackrabbit.oak.spi.security.privilege.PrivilegeConstants.JCR_ALL;
import static
org.apache.jackrabbit.oak.upgrade.cli.parser.OptionParserFactory.SKIP_NAME_CHECK;
-import static
org.apache.jackrabbit.oak.upgrade.nodestate.FilteringNodeState.ALL;
-import static
org.apache.jackrabbit.oak.upgrade.nodestate.FilteringNodeState.NONE;
-import static
org.apache.jackrabbit.oak.upgrade.nodestate.NodeStateCopier.copyProperties;
import java.io.File;
import java.io.IOException;
@@ -66,6 +63,7 @@ import com.google.common.collect.HashBiM
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Lists;
+import org.apache.jackrabbit.JcrConstants;
import org.apache.jackrabbit.api.security.authorization.PrivilegeManager;
import org.apache.jackrabbit.core.IndexAccessor;
import org.apache.jackrabbit.core.RepositoryContext;
@@ -82,6 +80,7 @@ import org.apache.jackrabbit.core.securi
import org.apache.jackrabbit.oak.api.CommitFailedException;
import org.apache.jackrabbit.oak.api.Root;
import org.apache.jackrabbit.oak.api.Tree;
+import org.apache.jackrabbit.oak.commons.PathUtils;
import org.apache.jackrabbit.oak.namepath.NamePathMapper;
import org.apache.jackrabbit.oak.plugins.index.CompositeIndexEditorProvider;
import org.apache.jackrabbit.oak.plugins.index.IndexEditorProvider;
@@ -90,11 +89,13 @@ import org.apache.jackrabbit.oak.plugins
import
org.apache.jackrabbit.oak.plugins.index.property.PropertyIndexEditorProvider;
import
org.apache.jackrabbit.oak.plugins.index.reference.ReferenceEditorProvider;
import org.apache.jackrabbit.oak.plugins.name.NamespaceConstants;
+import org.apache.jackrabbit.oak.plugins.nodetype.TypePredicate;
+import org.apache.jackrabbit.oak.plugins.nodetype.write.InitialContent;
+import org.apache.jackrabbit.oak.plugins.value.ValueFactoryImpl;
+import org.apache.jackrabbit.oak.spi.commit.EmptyHook;
import org.apache.jackrabbit.oak.plugins.name.Namespaces;
import org.apache.jackrabbit.oak.plugins.nodetype.TypeEditorProvider;
-import org.apache.jackrabbit.oak.plugins.nodetype.write.InitialContent;
import
org.apache.jackrabbit.oak.plugins.nodetype.write.ReadWriteNodeTypeManager;
-import org.apache.jackrabbit.oak.plugins.value.ValueFactoryImpl;
import org.apache.jackrabbit.oak.security.SecurityProviderImpl;
import org.apache.jackrabbit.oak.spi.commit.CommitHook;
import org.apache.jackrabbit.oak.spi.commit.CommitInfo;
@@ -114,10 +115,11 @@ import org.apache.jackrabbit.oak.spi.sec
import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
import org.apache.jackrabbit.oak.spi.state.NodeState;
import org.apache.jackrabbit.oak.spi.state.NodeStore;
+import org.apache.jackrabbit.oak.upgrade.nodestate.FilteringNodeState;
import org.apache.jackrabbit.oak.upgrade.nodestate.NameFilteringNodeState;
+import org.apache.jackrabbit.oak.upgrade.nodestate.NodeStateCopier;
import org.apache.jackrabbit.oak.upgrade.nodestate.report.LoggingReporter;
import org.apache.jackrabbit.oak.upgrade.nodestate.report.ReportingNodeState;
-import org.apache.jackrabbit.oak.upgrade.nodestate.NodeStateCopier;
import org.apache.jackrabbit.oak.upgrade.security.AuthorizableFolderEditor;
import org.apache.jackrabbit.oak.upgrade.security.GroupEditorProvider;
import org.apache.jackrabbit.oak.upgrade.security.RestrictionEditorProvider;
@@ -141,6 +143,9 @@ import org.apache.lucene.index.TermEnum;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import static
org.apache.jackrabbit.oak.upgrade.nodestate.FilteringNodeState.ALL;
+import static
org.apache.jackrabbit.oak.upgrade.nodestate.FilteringNodeState.NONE;
+import static
org.apache.jackrabbit.oak.upgrade.nodestate.NodeStateCopier.copyProperties;
import static
org.apache.jackrabbit.oak.upgrade.version.VersionCopier.copyVersionStorage;
import static
org.apache.jackrabbit.oak.upgrade.version.VersionHistoryUtil.getVersionStorage;
@@ -154,6 +159,10 @@ public class RepositoryUpgrade {
public static final Set<String> DEFAULT_EXCLUDE_PATHS = NONE;
+ public static final Set<String> DEFAULT_FRAGMENT_PATHS = NONE;
+
+ public static final Set<String> DEFAULT_EXCLUDE_FRAGMENTS = NONE;
+
public static final Set<String> DEFAULT_MERGE_PATHS = NONE;
/**
@@ -177,6 +186,16 @@ public class RepositoryUpgrade {
private Set<String> excludePaths = DEFAULT_EXCLUDE_PATHS;
/**
+ * Paths supporting fragments during the copy process. Empty by default.
+ */
+ private Set<String> fragmentPaths = DEFAULT_FRAGMENT_PATHS;
+
+ /**
+ * Fragments to exclude during the copy process. Empty by default.
+ */
+ private Set<String> excludeFragments = DEFAULT_EXCLUDE_FRAGMENTS;
+
+ /**
* Paths to merge during the copy process. Empty by default.
*/
private Set<String> mergePaths = DEFAULT_MERGE_PATHS;
@@ -333,6 +352,25 @@ public class RepositoryUpgrade {
}
/**
+ * Sets the paths that should support the fragments.
+ *
+ * @param fragmentPaths Paths that should support fragments.
+ */
+ public void setFragmentPaths(@Nonnull String... fragmentPaths) {
+ this.fragmentPaths = copyOf(checkNotNull(fragmentPaths));
+ }
+
+ /**
+ * Sets the name fragments that should be excluded when the source
repository
+ * is copied to the target repository.
+ *
+ * @param excludes Name fragments to be excluded from the copy.
+ */
+ public void setExcludeFragments(@Nonnull String... excludes) {
+ this.excludeFragments = copyOf(checkNotNull(excludes));
+ }
+
+ /**
* Sets the paths that should be merged when the source repository
* is copied to the target repository.
*
@@ -545,12 +583,26 @@ public class RepositoryUpgrade {
target.merge(targetBuilder, new LoggingCompositeHook(hooks,
source, overrideEarlyShutdown()), CommitInfo.EMPTY);
logger.info("Processing commit hooks completed in {}s ({})",
watch.elapsed(TimeUnit.SECONDS), watch);
+
+ removeVersions();
+
logger.debug("Repository upgrade completed.");
} catch (Exception e) {
throw new RepositoryException("Failed to copy content", e);
}
}
+ private void removeVersions() throws CommitFailedException {
+ NodeState root = target.getRoot();
+ NodeState wrappedRoot = FilteringNodeState.wrap("/", root,
includePaths, excludePaths, fragmentPaths, excludeFragments);
+ List<String> versionablesToStrip =
VersionHistoryUtil.getVersionableNodes(wrappedRoot, new TypePredicate(root,
JcrConstants.MIX_VERSIONABLE), versionCopyConfiguration.getVersionsMinDate());
+ if (!versionablesToStrip.isEmpty()) {
+ logger.info("Removing version histories for included paths");
+ NodeBuilder newRoot = VersionHistoryUtil.removeVersions(root,
versionablesToStrip);
+ target.merge(newRoot, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+ }
+ }
+
private boolean overrideEarlyShutdown() {
if (earlyShutdown == false) {
return false;
@@ -566,6 +618,10 @@ public class RepositoryUpgrade {
logger.info("Overriding early shutdown to false because of the
copy versions settings");
return false;
}
+ if (c.isCopyVersions() &&
target.getRoot().hasChildNode(JcrConstants.JCR_SYSTEM)) {
+ logger.info("Overriding early shutdown to false because the target
exists");
+ return false;
+ }
return true;
}
@@ -931,6 +987,8 @@ public class RepositoryUpgrade {
NodeStateCopier.builder()
.include(includes)
.exclude(excludes)
+ .supportFragment(fragmentPaths)
+ .excludeFragments(excludeFragments)
.merge(merges)
.copy(sourceRoot, targetRoot);
Modified:
jackrabbit/oak/branches/1.2/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/checkpoint/CheckpointRetriever.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.2/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/checkpoint/CheckpointRetriever.java?rev=1815901&r1=1815900&r2=1815901&view=diff
==============================================================================
---
jackrabbit/oak/branches/1.2/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/checkpoint/CheckpointRetriever.java
(original)
+++
jackrabbit/oak/branches/1.2/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/checkpoint/CheckpointRetriever.java
Tue Nov 21 10:03:13 2017
@@ -23,10 +23,11 @@ import com.google.common.collect.Iterabl
import com.google.common.collect.Lists;
import org.apache.jackrabbit.oak.plugins.document.DocumentCheckpointRetriever;
import org.apache.jackrabbit.oak.plugins.document.DocumentNodeStore;
+import org.apache.jackrabbit.oak.plugins.segment.CheckpointAccessor;
import org.apache.jackrabbit.oak.spi.state.ChildNodeEntry;
import org.apache.jackrabbit.oak.spi.state.NodeState;
import org.apache.jackrabbit.oak.spi.state.NodeStore;
-import org.apache.jackrabbit.oak.upgrade.cli.node.TarNodeStore;
+import org.apache.jackrabbit.oak.upgrade.cli.node.SegmentFactory;
import javax.annotation.Nullable;
import java.util.Collections;
@@ -62,11 +63,7 @@ public final class CheckpointRetriever {
@Override
public int compareTo(Checkpoint o) {
- return compare(this.created, o.created);
- }
-
- private static int compare(long x, long y) {
- return (x < y) ? -1 : ((x == y) ? 0 : 1);
+ return Long.valueOf(this.created).compareTo(o.created);
}
}
@@ -75,19 +72,21 @@ public final class CheckpointRetriever {
public static List<Checkpoint> getCheckpoints(NodeStore nodeStore) {
List<Checkpoint> result;
- if (nodeStore instanceof TarNodeStore) {
- result = getCheckpoints((TarNodeStore) nodeStore);
+ if (nodeStore instanceof
org.apache.jackrabbit.oak.plugins.segment.SegmentNodeStore) {
+ result =
getCheckpoints(org.apache.jackrabbit.oak.plugins.segment.CheckpointAccessor.getCheckpointsRoot((org.apache.jackrabbit.oak.plugins.segment.SegmentNodeStore)
nodeStore));
} else if (nodeStore instanceof DocumentNodeStore) {
result =
DocumentCheckpointRetriever.getCheckpoints((DocumentNodeStore) nodeStore);
+ } else if (nodeStore instanceof SegmentFactory.NodeStoreWithFileStore)
{
+ result =
getCheckpoints(CheckpointAccessor.getCheckpointsRoot(((SegmentFactory.NodeStoreWithFileStore)
nodeStore).getNodeStore()));
} else {
- result = Collections.emptyList();
+ return null;
}
Collections.sort(result);
return result;
}
- private static List<Checkpoint> getCheckpoints(TarNodeStore nodeStore) {
- return
Lists.newArrayList(Iterables.transform(nodeStore.getSuperRoot().getChildNode("checkpoints").getChildNodeEntries(),
new Function<ChildNodeEntry, Checkpoint>() {
+ private static List<Checkpoint> getCheckpoints(NodeState checkpointRoot) {
+ return
Lists.newArrayList(Iterables.transform(checkpointRoot.getChildNodeEntries(),
new Function<ChildNodeEntry, Checkpoint>() {
@Nullable
@Override
public Checkpoint apply(@Nullable ChildNodeEntry input) {
@@ -95,4 +94,4 @@ public final class CheckpointRetriever {
}
}));
}
-}
+}
\ No newline at end of file
Modified:
jackrabbit/oak/branches/1.2/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/MigrationFactory.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.2/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/MigrationFactory.java?rev=1815901&r1=1815900&r2=1815901&view=diff
==============================================================================
---
jackrabbit/oak/branches/1.2/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/MigrationFactory.java
(original)
+++
jackrabbit/oak/branches/1.2/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/MigrationFactory.java
Tue Nov 21 10:03:13 2017
@@ -18,6 +18,7 @@ package org.apache.jackrabbit.oak.upgrad
import java.io.IOException;
import java.util.Iterator;
+import java.util.List;
import java.util.ServiceLoader;
import javax.jcr.RepositoryException;
@@ -85,6 +86,12 @@ public class MigrationFactory {
if (options.getExcludePaths() != null) {
upgrade.setExcludes(options.getExcludePaths());
}
+ if (options.getFragmentPaths() != null) {
+ upgrade.setFragmentPaths(options.getFragmentPaths());
+ }
+ if (options.getExcludeFragments() != null) {
+ upgrade.setExcludeFragments(options.getExcludeFragments());
+ }
if (options.getMergePaths() != null) {
upgrade.setMerges(options.getMergePaths());
}
@@ -93,10 +100,7 @@ public class MigrationFactory {
upgrade.setSkipOnError(!options.isFailOnError());
upgrade.setEarlyShutdown(options.isEarlyShutdown());
upgrade.setSkipInitialization(options.isSkipInitialization());
- ServiceLoader<CommitHook> loader =
ServiceLoader.load(CommitHook.class);
- Iterator<CommitHook> iterator = loader.iterator();
- ImmutableList.Builder<CommitHook> builder = ImmutableList.<CommitHook>
builder().addAll(iterator);
- upgrade.setCustomCommitHooks(builder.build());
+ upgrade.setCustomCommitHooks(loacCommitHooks());
return upgrade;
}
@@ -110,14 +114,30 @@ public class MigrationFactory {
if (options.getExcludePaths() != null) {
sidegrade.setExcludes(options.getExcludePaths());
}
+ if (options.getExcludeFragments() != null) {
+ sidegrade.setExcludeFragments(options.getExcludeFragments());
+ }
+ if (options.getFragmentPaths() != null) {
+ sidegrade.setFragmentPaths(options.getFragmentPaths());
+ }
if (options.getMergePaths() != null) {
sidegrade.setMerges(options.getMergePaths());
}
sidegrade.setFilterLongNames(stores.getSrcType().isSupportLongNames()
&& !stores.getDstType().isSupportLongNames());
+ sidegrade.setIncludeIndex(options.isIncludeIndex());
sidegrade.setVerify(options.isVerify());
sidegrade.setOnlyVerify(options.isOnlyVerify());
sidegrade.setSkipCheckpoints(options.isSkipCheckpoints());
+ sidegrade.setForceCheckpoints(options.isForceCheckpoints());
+ sidegrade.setCustomCommitHooks(loacCommitHooks());
return sidegrade;
}
+ private List<CommitHook> loacCommitHooks() {
+ ServiceLoader<CommitHook> loader =
ServiceLoader.load(CommitHook.class);
+ Iterator<CommitHook> iterator = loader.iterator();
+ ImmutableList.Builder<CommitHook> builder = ImmutableList.<CommitHook>
builder().addAll(iterator);
+ return builder.build();
+ }
+
}
Modified:
jackrabbit/oak/branches/1.2/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/blob/S3DataStoreFactory.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.2/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/blob/S3DataStoreFactory.java?rev=1815901&r1=1815900&r2=1815901&view=diff
==============================================================================
---
jackrabbit/oak/branches/1.2/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/blob/S3DataStoreFactory.java
(original)
+++
jackrabbit/oak/branches/1.2/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/blob/S3DataStoreFactory.java
Tue Nov 21 10:03:13 2017
@@ -21,29 +21,26 @@ import java.io.File;
import java.io.FileReader;
import java.io.IOException;
import java.util.HashSet;
+import java.util.Map;
import java.util.Properties;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import javax.jcr.RepositoryException;
-import org.apache.commons.io.FileUtils;
+import com.google.common.collect.Maps;
import org.apache.commons.io.IOUtils;
-import org.apache.jackrabbit.aws.ext.ds.S3DataStore;
-import org.apache.jackrabbit.core.data.CachingDataStore;
import org.apache.jackrabbit.core.data.DataStoreException;
+import org.apache.jackrabbit.oak.blob.cloud.aws.s3.S3DataStore;
+import org.apache.jackrabbit.oak.commons.PropertiesUtil;
import org.apache.jackrabbit.oak.plugins.blob.datastore.DataStoreBlobStore;
import org.apache.jackrabbit.oak.spi.blob.BlobStore;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
import com.google.common.io.Closer;
import com.google.common.io.Files;
public class S3DataStoreFactory implements BlobStoreFactory {
- private static final Logger log =
LoggerFactory.getLogger(S3DataStoreFactory.class);
-
private static final Pattern STRIP_VALUE_PATTERN =
Pattern.compile("^[TILFDXSCB]?\"(.*)\"\\W*$");
private final Properties props;
@@ -63,11 +60,6 @@ public class S3DataStoreFactory implemen
IOUtils.closeQuietly(reader);
}
- for (Object key : new HashSet<Object>(props.keySet())) {
- String value = props.getProperty((String) key);
- props.put(key, stripValue(value));
- }
-
this.directory = directory;
this.tempHomeDir = Files.createTempDir();
this.ignoreMissingBlobs = ignoreMissingBlobs;
@@ -75,15 +67,14 @@ public class S3DataStoreFactory implemen
@Override
public BlobStore create(Closer closer) throws IOException {
- S3DataStore delegate = new S3DataStore();
- delegate.setProperties(props);
- delegate.setPath(directory);
+ S3DataStore delegate = createDS(directory, props);
+
try {
delegate.init(tempHomeDir.getPath());
} catch (RepositoryException e) {
throw new IOException(e);
}
- closer.register(asCloseable(delegate, tempHomeDir));
+ closer.register(asCloseable(delegate));
if (ignoreMissingBlobs) {
return new SafeDataStoreBlobStore(delegate);
} else {
@@ -91,21 +82,40 @@ public class S3DataStoreFactory implemen
}
}
- private static Closeable asCloseable(final CachingDataStore store, final
File tempHomeDir) {
+ static S3DataStore createDS(String directory, Properties props) {
+ Properties strippedProperties = new Properties();
+ Map<String, String> map = Maps.newHashMap();
+
+ for (Object key : new HashSet<Object>(props.keySet())) {
+ String strippedValue = stripValue(props.getProperty((String) key));
+
+ strippedProperties.put(key, strippedValue);
+ map.put((String) key, strippedValue);
+ }
+
+ S3DataStore ds = new S3DataStore();
+ ds.setProperties(strippedProperties);
+ ds.setPath(directory);
+ PropertiesUtil.populate(ds, map, false);
+ return ds;
+ }
+
+ private static Closeable asCloseable(final S3DataStore store) {
return new Closeable() {
@Override
public void close() throws IOException {
try {
while (!store.getPendingUploads().isEmpty()) {
- log.info("Waiting for following uploads to finish: " +
store.getPendingUploads());
- Thread.sleep(1000);
+ Thread.sleep(100);
}
- store.close();
- FileUtils.deleteDirectory(tempHomeDir);
- } catch (DataStoreException e) {
- throw new IOException(e);
} catch (InterruptedException e) {
throw new IOException(e);
+ } finally {
+ try {
+ store.close();
+ } catch (DataStoreException e) {
+ throw new IOException(e);
+ }
}
}
};
Modified:
jackrabbit/oak/branches/1.2/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/node/SegmentFactory.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.2/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/node/SegmentFactory.java?rev=1815901&r1=1815900&r2=1815901&view=diff
==============================================================================
---
jackrabbit/oak/branches/1.2/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/node/SegmentFactory.java
(original)
+++
jackrabbit/oak/branches/1.2/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/node/SegmentFactory.java
Tue Nov 21 10:03:13 2017
@@ -25,10 +25,10 @@ import org.apache.jackrabbit.oak.plugins
import org.apache.jackrabbit.oak.plugins.segment.file.FileStore;
import org.apache.jackrabbit.oak.plugins.segment.file.FileStore.Builder;
import org.apache.jackrabbit.oak.spi.blob.BlobStore;
-import org.apache.jackrabbit.oak.spi.state.NodeState;
import org.apache.jackrabbit.oak.spi.state.NodeStore;
import com.google.common.io.Closer;
+import org.apache.jackrabbit.oak.spi.state.ProxyNodeStore;
import javax.annotation.Nullable;
@@ -78,12 +78,7 @@ public class SegmentFactory implements N
closer.register(asCloseable(fs));
- return new TarNodeStore(new SegmentNodeStore(fs), new
TarNodeStore.SuperRootProvider() {
- @Override
- public NodeState getSuperRoot() {
- return fs.getHead();
- }
- });
+ return new NodeStoreWithFileStore(new SegmentNodeStore(fs), fs);
}
@Override
@@ -126,4 +121,26 @@ public class SegmentFactory implements N
private static class ExternalBlobFound extends RuntimeException {
}
+
+ public static class NodeStoreWithFileStore extends ProxyNodeStore {
+
+ private final SegmentNodeStore segmentNodeStore;
+
+ private final FileStore fileStore;
+
+ public NodeStoreWithFileStore(SegmentNodeStore segmentNodeStore,
FileStore fileStore) {
+ this.segmentNodeStore = segmentNodeStore;
+ this.fileStore = fileStore;
+ }
+
+ public FileStore getFileStore() {
+ return fileStore;
+ }
+
+ @Override
+ public SegmentNodeStore getNodeStore() {
+ return segmentNodeStore;
+ }
+ }
+
}
Modified:
jackrabbit/oak/branches/1.2/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/parser/MigrationCliArguments.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.2/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/parser/MigrationCliArguments.java?rev=1815901&r1=1815900&r2=1815901&view=diff
==============================================================================
---
jackrabbit/oak/branches/1.2/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/parser/MigrationCliArguments.java
(original)
+++
jackrabbit/oak/branches/1.2/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/parser/MigrationCliArguments.java
Tue Nov 21 10:03:13 2017
@@ -52,6 +52,10 @@ public final class MigrationCliArguments
return (Integer) options.valueOf(optionName);
}
+ public Boolean getBooleanOption(String optionName) {
+ return (Boolean) options.valueOf(optionName);
+ }
+
public String[] getOptionList(String optionName) {
String option = getOption(optionName);
if (option == null) {
Modified:
jackrabbit/oak/branches/1.2/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/parser/MigrationOptions.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.2/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/parser/MigrationOptions.java?rev=1815901&r1=1815900&r2=1815901&view=diff
==============================================================================
---
jackrabbit/oak/branches/1.2/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/parser/MigrationOptions.java
(original)
+++
jackrabbit/oak/branches/1.2/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/parser/MigrationOptions.java
Tue Nov 21 10:03:13 2017
@@ -46,8 +46,14 @@ public class MigrationOptions {
private final String[] excludePaths;
+ private final String[] fragmentPaths;
+
+ private final String[] excludeFragments;
+
private final String[] mergePaths;
+ private final boolean includeIndex;
+
private final boolean failOnError;
private final boolean earlyShutdown;
@@ -64,6 +70,8 @@ public class MigrationOptions {
private final boolean skipCheckpoints;
+ private final boolean forceCheckpoints;
+
private final String srcUser;
private final String srcPassword;
@@ -113,7 +121,10 @@ public class MigrationOptions {
}
this.includePaths =
checkPaths(args.getOptionList(OptionParserFactory.INCLUDE_PATHS));
this.excludePaths =
checkPaths(args.getOptionList(OptionParserFactory.EXCLUDE_PATHS));
+ this.fragmentPaths =
checkPaths(args.getOptionList(OptionParserFactory.FRAGMENT_PATHS));
+ this.excludeFragments =
args.getOptionList(OptionParserFactory.EXCLUDE_FRAGMENTS);
this.mergePaths =
checkPaths(args.getOptionList(OptionParserFactory.MERGE_PATHS));
+ this.includeIndex = args.hasOption(OptionParserFactory.INCLUDE_INDEX);
this.failOnError = args.hasOption(OptionParserFactory.FAIL_ON_ERROR);
this.earlyShutdown =
args.hasOption(OptionParserFactory.EARLY_SHUTDOWN);
this.skipInitialization =
args.hasOption(OptionParserFactory.SKIP_INIT);
@@ -122,6 +133,7 @@ public class MigrationOptions {
this.verify = args.hasOption(OptionParserFactory.VERIFY);
this.onlyVerify = args.hasOption(OptionParserFactory.ONLY_VERIFY);
this.skipCheckpoints =
args.hasOption(OptionParserFactory.SKIP_CHECKPOINTS);
+ this.forceCheckpoints =
args.hasOption(OptionParserFactory.FORCE_CHECKPOINTS);
this.srcUser = args.getOption(OptionParserFactory.SRC_USER);
this.srcPassword = args.getOption(OptionParserFactory.SRC_USER);
@@ -139,8 +151,7 @@ public class MigrationOptions {
this.dstS3Config = args.getOption(OptionParserFactory.DST_S3_CONFIG);
if (args.hasOption(OptionParserFactory.SRC_EXTERNAL_BLOBS)) {
- this.srcExternalBlobs = Boolean
- .valueOf(OptionParserFactory.SRC_EXTERNAL_BLOBS);
+ this.srcExternalBlobs =
args.getBooleanOption(OptionParserFactory.SRC_EXTERNAL_BLOBS);
} else {
this.srcExternalBlobs = null;
}
@@ -174,6 +185,14 @@ public class MigrationOptions {
return excludePaths;
}
+ public String[] getFragmentPaths() {
+ return fragmentPaths;
+ }
+
+ public String[] getExcludeFragments() {
+ return excludeFragments;
+ }
+
public String[] getMergePaths() {
return mergePaths;
}
@@ -194,6 +213,10 @@ public class MigrationOptions {
return skipNameCheck;
}
+ public boolean isIncludeIndex() {
+ return includeIndex;
+ }
+
public boolean isIgnoreMissingBinaries() {
return ignoreMissingBinaries;
}
@@ -210,6 +233,10 @@ public class MigrationOptions {
return skipCheckpoints;
}
+ public boolean isForceCheckpoints() {
+ return forceCheckpoints;
+ }
+
public String getSrcUser() {
return srcUser;
}
@@ -315,6 +342,14 @@ public class MigrationOptions {
log.info("paths to exclude: {}", (Object) excludePaths);
}
+ if (fragmentPaths != null) {
+ log.info("paths supporting fragments: {}", (Object) fragmentPaths);
+ }
+
+ if (excludeFragments != null) {
+ log.info("fragments to exclude: {}", (Object) excludeFragments);
+ }
+
if (failOnError) {
log.info("Unreadable nodes will cause failure of the entire
transaction");
}
@@ -331,6 +366,10 @@ public class MigrationOptions {
log.info("Test for long-named nodes will be disabled");
}
+ if (includeIndex) {
+ log.info("Index data for the paths {} will be copied", (Object)
includePaths);
+ }
+
if (ignoreMissingBinaries) {
log.info("Missing binaries won't break the migration");
}
@@ -343,6 +382,10 @@ public class MigrationOptions {
log.info("Checkpoints won't be migrated");
}
+ if (forceCheckpoints) {
+ log.info("Checkpoints will be migrated even with the custom paths
specified");
+ }
+
log.info("Cache size: {} MB", cacheSizeInMB);
}
Modified:
jackrabbit/oak/branches/1.2/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/parser/OptionParserFactory.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.2/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/parser/OptionParserFactory.java?rev=1815901&r1=1815900&r2=1815901&view=diff
==============================================================================
---
jackrabbit/oak/branches/1.2/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/parser/OptionParserFactory.java
(original)
+++
jackrabbit/oak/branches/1.2/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/parser/OptionParserFactory.java
Tue Nov 21 10:03:13 2017
@@ -70,18 +70,26 @@ public class OptionParserFactory {
public static final String EXCLUDE_PATHS = "exclude-paths";
+ public static final String FRAGMENT_PATHS = "fragment-paths";
+
+ public static final String EXCLUDE_FRAGMENTS = "exclude-fragments";
+
public static final String MERGE_PATHS = "merge-paths";
public static final String SKIP_INIT = "skip-init";
public static final String SKIP_NAME_CHECK = "skip-name-check";
+ public static final String INCLUDE_INDEX = "include-index";
+
public static final String VERIFY = "verify";
public static final String ONLY_VERIFY = "only-verify";
public static final String SKIP_CHECKPOINTS = "skip-checkpoints";
+ public static final String FORCE_CHECKPOINTS = "force-checkpoints";
+
public static OptionParser create() {
OptionParser op = new OptionParser();
addUsageOptions(op);
@@ -114,7 +122,8 @@ public class OptionParserFactory {
op.accepts(DST_S3_CONFIG, "Configuration file for the target
S3DataStore").withRequiredArg()
.ofType(String.class);
op.accepts(IGNORE_MISSING_BINARIES, "Don't break the migration if some
binaries are missing");
- op.accepts(SRC_EXTERNAL_BLOBS, "Flag specifying if the source Store
has external references or not");
+ op.accepts(SRC_EXTERNAL_BLOBS, "Flag specifying if the source Store
has external references or not")
+ .withRequiredArg().ofType(Boolean.class);
}
private static void addRdbOptions(OptionParser op) {
@@ -129,8 +138,13 @@ public class OptionParserFactory {
.ofType(String.class);
op.accepts(EXCLUDE_PATHS, "Comma-separated list of paths to exclude
during copy.").withRequiredArg()
.ofType(String.class);
+ op.accepts(FRAGMENT_PATHS, "Comma-separated list of paths supporting
fragments.").withRequiredArg()
+ .ofType(String.class);
+ op.accepts(EXCLUDE_FRAGMENTS, "Comma-separated list of fragments to
exclude during copy.").withRequiredArg()
+ .ofType(String.class);
op.accepts(MERGE_PATHS, "Comma-separated list of paths to merge during
copy.").withRequiredArg()
.ofType(String.class);
+ op.accepts(INCLUDE_INDEX, "Copy index data for paths specified in the
" + INCLUDE_PATHS + " option");
}
private static void addVersioningOptions(OptionParser op) {
@@ -153,5 +167,6 @@ public class OptionParserFactory {
op.accepts(VERIFY, "After the sidegrade check whether the source
repository is exactly the same as destination");
op.accepts(ONLY_VERIFY, "Performs only --" + VERIFY + ", without
copying content");
op.accepts(SKIP_CHECKPOINTS, "Don't copy checkpoints on the full
segment->segment migration");
+ op.accepts(FORCE_CHECKPOINTS, "Copy checkpoints even if the
--include,exclude,merge-paths option is specified");
}
}
Modified:
jackrabbit/oak/branches/1.2/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/parser/StoreArguments.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.2/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/parser/StoreArguments.java?rev=1815901&r1=1815900&r2=1815901&view=diff
==============================================================================
---
jackrabbit/oak/branches/1.2/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/parser/StoreArguments.java
(original)
+++
jackrabbit/oak/branches/1.2/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/parser/StoreArguments.java
Tue Nov 21 10:03:13 2017
@@ -39,6 +39,8 @@ public class StoreArguments {
private static final String REPOSITORY_XML = "repository.xml";
+ public static final String SEGMENT_OLD_PREFIX = "segment-old:";
+
private static final Logger log =
LoggerFactory.getLogger(StoreArguments.class);
private final MigrationOptions options;
Modified:
jackrabbit/oak/branches/1.2/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/nodestate/AbstractDecoratedNodeState.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.2/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/nodestate/AbstractDecoratedNodeState.java?rev=1815901&r1=1815900&r2=1815901&view=diff
==============================================================================
---
jackrabbit/oak/branches/1.2/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/nodestate/AbstractDecoratedNodeState.java
(original)
+++
jackrabbit/oak/branches/1.2/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/nodestate/AbstractDecoratedNodeState.java
Tue Nov 21 10:03:13 2017
@@ -36,6 +36,7 @@ import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import java.util.ArrayList;
import java.util.Collection;
+import java.util.Collections;
import static com.google.common.base.Predicates.notNull;
import static
org.apache.jackrabbit.oak.plugins.tree.impl.TreeConstants.OAK_CHILD_ORDER;
@@ -48,6 +49,10 @@ public abstract class AbstractDecoratedN
this.delegate = delegate;
}
+ public NodeState getDelegate() {
+ return delegate;
+ }
+
protected boolean hideChild(@Nonnull final String name, @Nonnull final
NodeState delegateChild) {
return false;
}
@@ -64,6 +69,11 @@ public abstract class AbstractDecoratedN
return false;
}
+ @Nonnull
+ protected Iterable<PropertyState> getNewPropertyStates() {
+ return Collections.emptyList();
+ }
+
@CheckForNull
protected abstract PropertyState decorateProperty(@Nonnull final
PropertyState delegatePropertyState);
@@ -149,7 +159,16 @@ public abstract class AbstractDecoratedN
@Override
@CheckForNull
public PropertyState getProperty(@Nonnull String name) {
- return decorate(delegate.getProperty(name));
+ PropertyState ps = decorate(delegate.getProperty(name));
+ if (ps == null) {
+ for (PropertyState p : getNewPropertyStates()) {
+ if (name.equals(p.getName())) {
+ ps = p;
+ break;
+ }
+ }
+ }
+ return ps;
}
@Override
@@ -165,7 +184,7 @@ public abstract class AbstractDecoratedN
}
}
);
- return Iterables.filter(propertyStates, notNull());
+ return Iterables.filter(Iterables.concat(propertyStates,
getNewPropertyStates()), notNull());
}
/**
Modified:
jackrabbit/oak/branches/1.2/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/nodestate/FilteringNodeState.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.2/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/nodestate/FilteringNodeState.java?rev=1815901&r1=1815900&r2=1815901&view=diff
==============================================================================
---
jackrabbit/oak/branches/1.2/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/nodestate/FilteringNodeState.java
(original)
+++
jackrabbit/oak/branches/1.2/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/nodestate/FilteringNodeState.java
Tue Nov 21 10:03:13 2017
@@ -67,6 +67,10 @@ public class FilteringNodeState extends
private final Set<String> excludedPaths;
+ private final Set<String> fragmentPaths;
+
+ private final Set<String> excludedFragments;
+
/**
* Factory method that conditionally decorates the given node-state
* iff the node-state is (a) hidden itself or (b) has hidden descendants.
@@ -75,6 +79,8 @@ public class FilteringNodeState extends
* @param delegate The node-state to decorate.
* @param includePaths A Set of paths that should be visible. Defaults to
["/"] if {@code null}.
* @param excludePaths A Set of paths that should be hidden. Empty if
{@code null}.
+ * @param fragmentPaths A Set of paths that should support the fragments
(see below). Empty if {@code null}.
+ * @param excludedFragments A Set of name fragments that should be hidden.
Empty if {@code null}.
* @return The decorated node-state if required, the original node-state
if decoration is unnecessary.
*/
@Nonnull
@@ -82,12 +88,16 @@ public class FilteringNodeState extends
@Nonnull final String path,
@Nonnull final NodeState delegate,
@Nullable final Set<String> includePaths,
- @Nullable final Set<String> excludePaths
+ @Nullable final Set<String> excludePaths,
+ @Nullable final Set<String> fragmentPaths,
+ @Nullable final Set<String> excludedFragments
) {
final Set<String> includes = defaultIfEmpty(includePaths, ALL);
final Set<String> excludes = defaultIfEmpty(excludePaths, NONE);
- if (hasHiddenDescendants(path, includes, excludes)) {
- return new FilteringNodeState(path, delegate, includes, excludes);
+ final Set<String> safeFragmentPaths = defaultIfEmpty(fragmentPaths,
NONE);
+ final Set<String> safeExcludedFragments =
defaultIfEmpty(excludedFragments, NONE);
+ if (hasHiddenDescendants(path, includes, excludes, safeFragmentPaths,
safeExcludedFragments)) {
+ return new FilteringNodeState(path, delegate, includes, excludes,
fragmentPaths, safeExcludedFragments);
}
return delegate;
}
@@ -96,24 +106,28 @@ public class FilteringNodeState extends
@Nonnull final String path,
@Nonnull final NodeState delegate,
@Nonnull final Set<String> includedPaths,
- @Nonnull final Set<String> excludedPaths
+ @Nonnull final Set<String> excludedPaths,
+ @Nonnull final Set<String> fragmentPaths,
+ @Nonnull final Set<String> excludedFragments
) {
super(delegate);
this.path = path;
this.includedPaths = includedPaths;
this.excludedPaths = excludedPaths;
+ this.fragmentPaths = fragmentPaths;
+ this.excludedFragments = excludedFragments;
}
@Nonnull
@Override
protected NodeState decorateChild(@Nonnull final String name, @Nonnull
final NodeState child) {
final String childPath = PathUtils.concat(path, name);
- return wrap(childPath, child, includedPaths, excludedPaths);
+ return wrap(childPath, child, includedPaths, excludedPaths,
fragmentPaths, excludedFragments);
}
@Override
protected boolean hideChild(@Nonnull final String name, @Nonnull final
NodeState delegateChild) {
- return isHidden(PathUtils.concat(path, name), includedPaths,
excludedPaths);
+ return isHidden(PathUtils.concat(path, name), includedPaths,
excludedPaths, excludedFragments);
}
@Override
@@ -128,14 +142,16 @@ public class FilteringNodeState extends
* @param path Path to be checked
* @param includes Include paths
* @param excludes Exclude paths
+ * @param excludedFragments Exclude fragments
* @return Whether the {@code path} is hidden or not.
*/
private static boolean isHidden(
@Nonnull final String path,
@Nonnull final Set<String> includes,
- @Nonnull final Set<String> excludes
+ @Nonnull final Set<String> excludes,
+ @Nonnull final Set<String> excludedFragments
) {
- return isExcluded(path, excludes) || !isIncluded(path, includes);
+ return isExcluded(path, excludes, excludedFragments) ||
!isIncluded(path, includes);
}
/**
@@ -145,14 +161,20 @@ public class FilteringNodeState extends
* @param path Path to be checked
* @param includePaths Include paths
* @param excludePaths Exclude paths
+ * @param excludedFragments Exclude fragments
* @return Whether the {@code path} or any of its descendants are hidden
or not.
*/
private static boolean hasHiddenDescendants(
@Nonnull final String path,
@Nonnull final Set<String> includePaths,
- @Nonnull final Set<String> excludePaths
+ @Nonnull final Set<String> excludePaths,
+ @Nonnull final Set<String> fragmentPaths,
+ @Nonnull final Set<String> excludedFragments
) {
- return isHidden(path, includePaths, excludePaths)
+ return isHidden(path, includePaths, excludePaths, excludedFragments)
+ || isAncestorOfAnyPath(path, fragmentPaths)
+ || isDescendantOfAnyPath(path, fragmentPaths)
+ || fragmentPaths.contains(path)
|| isAncestorOfAnyPath(path, excludePaths)
|| isAncestorOfAnyPath(path, includePaths);
}
@@ -181,10 +203,11 @@ public class FilteringNodeState extends
*
* @param path Path to be checked
* @param excludePaths Exclude paths
+ * @param excludedFragments Exclude fragments
* @return Whether the path is covered by the excldue paths or not.
*/
- private static boolean isExcluded(@Nonnull final String path, @Nonnull
final Set<String> excludePaths) {
- return excludePaths.contains(path) || isDescendantOfAnyPath(path,
excludePaths);
+ private static boolean isExcluded(@Nonnull final String path, @Nonnull
final Set<String> excludePaths, @Nonnull final Set<String> excludedFragments) {
+ return excludePaths.contains(path) || isDescendantOfAnyPath(path,
excludePaths) || containsAnyFragment(path, excludedFragments);
}
/**
@@ -218,6 +241,22 @@ public class FilteringNodeState extends
return true;
}
}
+ return false;
+ }
+
+ /**
+ * Utility method to check whether the passed path contains any of the
provided {@code fragments}.
+ *
+ * @param path Path
+ * @param fragments Fragments, which the path may contain
+ * @return true if {@code path} contains any of the {@code fragments},
false otherwise.
+ */
+ private static boolean containsAnyFragment(@Nonnull final String path,
@Nonnull final Set<String> fragments) {
+ for (final String f : fragments) {
+ if (path.contains(f)) {
+ return true;
+ }
+ }
return false;
}
