Author: tomekr
Date: Tue Nov 21 08:33:10 2017
New Revision: 1815885
URL: http://svn.apache.org/viewvc?rev=1815885&view=rev
Log:
OAK-5933: Checkpoints are not sorted correctly in RepositorySidegrade
OAK-6003: Allow to migrate checkpoints for all type of sidegrades
OAK-6011: Test failure: JdbcToSegmentTest:validateMigration
OAK-6057: incorrect system property check in blob/upgrade tests
OAK-6131: No need to rebuild the counter/uuid index anymore
OAK-6188: Allow to exclude nodes containing name fragment during the migration
OAK-6190: Allow to migrate checkpoints even if the custom include paths are
specified
OAK-6306: Upgrade uses lucene wrong version (transient dependency)
OAK-6336: Use the oak-blob-cloud for s3 support in oak-upgrade
OAK-6360: Extra diagnostic info for the checkpoint migration
OAK-6454: Inaccurate data in the oak-upgrade progress logger
OAK-6560: Sidegrade uses too much memory
OAK-6595: Allow to include indexing data for the included paths
OAK-6633: Overwriting a versionable node with the copy-versions=false doesn't
remove versionable properties
OAK-6878: Populate S3DataStore fields with the passed properties
Added:
jackrabbit/oak/branches/1.6/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/plugins/
jackrabbit/oak/branches/1.6/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/plugins/document/
jackrabbit/oak/branches/1.6/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentCheckpointRetriever.java
jackrabbit/oak/branches/1.6/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/plugins/segment/
jackrabbit/oak/branches/1.6/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/plugins/segment/CheckpointAccessor.java
jackrabbit/oak/branches/1.6/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/segment/
jackrabbit/oak/branches/1.6/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/segment/CheckpointAccessor.java
jackrabbit/oak/branches/1.6/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/PersistingDiff.java
jackrabbit/oak/branches/1.6/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/checkpoint/
jackrabbit/oak/branches/1.6/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/checkpoint/CheckpointRetriever.java
jackrabbit/oak/branches/1.6/oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/cli/JdbcToSegmentWithMetadataTest.java
- copied, changed from r1815783,
jackrabbit/oak/branches/1.6/oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/cli/JdbcToSegmentTest.java
Removed:
jackrabbit/oak/branches/1.6/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/node/TarNodeStore.java
Modified:
jackrabbit/oak/branches/1.6/oak-upgrade/pom.xml
jackrabbit/oak/branches/1.6/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/JackrabbitNodeState.java
jackrabbit/oak/branches/1.6/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/RepositorySidegrade.java
jackrabbit/oak/branches/1.6/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/RepositoryUpgrade.java
jackrabbit/oak/branches/1.6/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/MigrationFactory.java
jackrabbit/oak/branches/1.6/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/node/JdbcFactory.java
jackrabbit/oak/branches/1.6/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/node/MongoFactory.java
jackrabbit/oak/branches/1.6/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/node/SegmentFactory.java
jackrabbit/oak/branches/1.6/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/node/SegmentTarFactory.java
jackrabbit/oak/branches/1.6/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/parser/MigrationCliArguments.java
jackrabbit/oak/branches/1.6/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/parser/MigrationOptions.java
jackrabbit/oak/branches/1.6/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/parser/OptionParserFactory.java
jackrabbit/oak/branches/1.6/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/nodestate/AbstractDecoratedNodeState.java
jackrabbit/oak/branches/1.6/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/nodestate/FilteringNodeState.java
jackrabbit/oak/branches/1.6/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/nodestate/NameFilteringNodeState.java
jackrabbit/oak/branches/1.6/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/nodestate/NodeStateCopier.java
jackrabbit/oak/branches/1.6/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/nodestate/report/LoggingReporter.java
jackrabbit/oak/branches/1.6/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/nodestate/report/PeriodicReporter.java
jackrabbit/oak/branches/1.6/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/version/VersionCopier.java
jackrabbit/oak/branches/1.6/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/version/VersionHistoryUtil.java
jackrabbit/oak/branches/1.6/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/version/VersionableEditor.java
jackrabbit/oak/branches/1.6/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/version/VersionablePropertiesEditor.java
jackrabbit/oak/branches/1.6/oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/CopyVersionHistorySidegradeTest.java
jackrabbit/oak/branches/1.6/oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/CopyVersionHistoryTest.java
jackrabbit/oak/branches/1.6/oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/IncludeExcludeSidegradeTest.java
jackrabbit/oak/branches/1.6/oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/IncludeExcludeUpgradeTest.java
jackrabbit/oak/branches/1.6/oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/RepeatedRepositorySidegradeTest.java
jackrabbit/oak/branches/1.6/oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/cli/AbstractOak2OakTest.java
jackrabbit/oak/branches/1.6/oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/cli/JdbcToSegmentTest.java
jackrabbit/oak/branches/1.6/oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/cli/MongoToMongoFbsTest.java
jackrabbit/oak/branches/1.6/oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/cli/MongoToMongoFdsTest.java
jackrabbit/oak/branches/1.6/oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/cli/MongoToSegmentTest.java
jackrabbit/oak/branches/1.6/oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/cli/SegmentToJdbcTest.java
jackrabbit/oak/branches/1.6/oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/cli/SegmentToMongoTest.java
jackrabbit/oak/branches/1.6/oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/cli/blob/FbsToS3Test.java
jackrabbit/oak/branches/1.6/oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/cli/blob/S3ToFbsTest.java
jackrabbit/oak/branches/1.6/oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/cli/container/SegmentTarNodeStoreContainer.java
jackrabbit/oak/branches/1.6/oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/nodestate/FilteringNodeStateTest.java
Modified: jackrabbit/oak/branches/1.6/oak-upgrade/pom.xml
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.6/oak-upgrade/pom.xml?rev=1815885&r1=1815884&r2=1815885&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.6/oak-upgrade/pom.xml (original)
+++ jackrabbit/oak/branches/1.6/oak-upgrade/pom.xml Tue Nov 21 08:33:10 2017
@@ -97,6 +97,13 @@
<version>${jackrabbit.version}</version>
</dependency>
+ <!-- it has to match the version used in jackrabbit-core -->
+ <dependency>
+ <groupId>org.apache.lucene</groupId>
+ <artifactId>lucene-core</artifactId>
+ <version>3.6.0</version>
+ </dependency>
+
<dependency>
<groupId>commons-lang</groupId>
<artifactId>commons-lang</artifactId>
Added:
jackrabbit/oak/branches/1.6/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentCheckpointRetriever.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.6/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentCheckpointRetriever.java?rev=1815885&view=auto
==============================================================================
---
jackrabbit/oak/branches/1.6/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentCheckpointRetriever.java
(added)
+++
jackrabbit/oak/branches/1.6/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentCheckpointRetriever.java
Tue Nov 21 08:33:10 2017
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.jackrabbit.oak.plugins.document;
+
+import org.apache.jackrabbit.oak.upgrade.checkpoint.CheckpointRetriever;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+public class DocumentCheckpointRetriever {
+
+ public static List<CheckpointRetriever.Checkpoint>
getCheckpoints(DocumentNodeStore nodeStore) {
+ List<CheckpointRetriever.Checkpoint> result = new ArrayList<>();
+ for (Map.Entry<Revision, Checkpoints.Info> e :
nodeStore.getCheckpoints().getCheckpoints().entrySet()) {
+ long created = e.getKey().getTimestamp();
+ long expiryTime = e.getValue().getExpiryTime();
+ String name = e.getKey().toString();
+ result.add(new CheckpointRetriever.Checkpoint(name, created,
expiryTime));
+ }
+ return result;
+ }
+
+}
\ No newline at end of file
Added:
jackrabbit/oak/branches/1.6/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/plugins/segment/CheckpointAccessor.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.6/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/plugins/segment/CheckpointAccessor.java?rev=1815885&view=auto
==============================================================================
---
jackrabbit/oak/branches/1.6/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/plugins/segment/CheckpointAccessor.java
(added)
+++
jackrabbit/oak/branches/1.6/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/plugins/segment/CheckpointAccessor.java
Tue Nov 21 08:33:10 2017
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.jackrabbit.oak.plugins.segment;
+
+import org.apache.jackrabbit.oak.plugins.segment.SegmentNodeStore;
+import org.apache.jackrabbit.oak.spi.state.NodeState;
+
+public final class CheckpointAccessor {
+
+ private CheckpointAccessor() {
+ }
+
+ public static NodeState getCheckpointsRoot(SegmentNodeStore
segmentNodeStore) {
+ return segmentNodeStore.getCheckpoints();
+ }
+
+}
Added:
jackrabbit/oak/branches/1.6/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/segment/CheckpointAccessor.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.6/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/segment/CheckpointAccessor.java?rev=1815885&view=auto
==============================================================================
---
jackrabbit/oak/branches/1.6/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/segment/CheckpointAccessor.java
(added)
+++
jackrabbit/oak/branches/1.6/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/segment/CheckpointAccessor.java
Tue Nov 21 08:33:10 2017
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.jackrabbit.oak.segment;
+
+import org.apache.jackrabbit.oak.spi.state.NodeState;
+
+public final class CheckpointAccessor {
+
+ private CheckpointAccessor() {
+ }
+
+ public static NodeState getCheckpointsRoot(SegmentNodeStore
segmentNodeStore) {
+ return segmentNodeStore.getCheckpoints();
+ }
+
+}
Modified:
jackrabbit/oak/branches/1.6/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/JackrabbitNodeState.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.6/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/JackrabbitNodeState.java?rev=1815885&r1=1815884&r2=1815885&view=diff
==============================================================================
---
jackrabbit/oak/branches/1.6/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/JackrabbitNodeState.java
(original)
+++
jackrabbit/oak/branches/1.6/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/JackrabbitNodeState.java
Tue Nov 21 08:33:10 2017
@@ -20,7 +20,6 @@ import static com.google.common.base.Pre
import static com.google.common.base.Preconditions.checkNotNull;
import static com.google.common.base.Preconditions.checkState;
import static com.google.common.collect.Iterables.addAll;
-import static com.google.common.collect.Lists.newArrayList;
import static com.google.common.collect.Lists.newArrayListWithCapacity;
import static com.google.common.collect.Maps.newHashMap;
import static com.google.common.collect.Maps.newLinkedHashMap;
@@ -63,7 +62,6 @@ import javax.jcr.RepositoryException;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.Lists;
import org.apache.jackrabbit.api.ReferenceBinary;
import org.apache.jackrabbit.core.RepositoryContext;
import org.apache.jackrabbit.core.id.NodeId;
Added:
jackrabbit/oak/branches/1.6/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/PersistingDiff.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.6/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/PersistingDiff.java?rev=1815885&view=auto
==============================================================================
---
jackrabbit/oak/branches/1.6/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/PersistingDiff.java
(added)
+++
jackrabbit/oak/branches/1.6/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/PersistingDiff.java
Tue Nov 21 08:33:10 2017
@@ -0,0 +1,206 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.jackrabbit.oak.upgrade;
+
+import com.google.common.collect.Lists;
+import org.apache.jackrabbit.oak.api.PropertyState;
+import org.apache.jackrabbit.oak.plugins.memory.MemoryNodeBuilder;
+import org.apache.jackrabbit.oak.segment.SegmentNodeState;
+import org.apache.jackrabbit.oak.segment.SegmentReader;
+import org.apache.jackrabbit.oak.segment.SegmentWriter;
+import org.apache.jackrabbit.oak.segment.file.FileStore;
+import org.apache.jackrabbit.oak.spi.blob.BlobStore;
+import org.apache.jackrabbit.oak.spi.state.NodeState;
+import org.apache.jackrabbit.oak.spi.state.NodeStateDiff;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.annotation.CheckForNull;
+import javax.annotation.Nonnull;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+import static com.google.common.base.Preconditions.checkState;
+import static
org.apache.jackrabbit.oak.plugins.memory.EmptyNodeState.EMPTY_NODE;
+
+public class PersistingDiff implements NodeStateDiff {
+
+ private static final Logger LOG =
LoggerFactory.getLogger(PersistingDiff.class);
+
+ /**
+ * Number of content updates that need to happen before the updates
+ * are automatically purged to the underlying segments.
+ */
+ private static final int UPDATE_LIMIT =
+ Integer.getInteger("upgrade.update.limit", 10000);
+
+ private final SegmentWriter writer;
+
+ private final SegmentReader reader;
+
+ private final BlobStore blobStore;
+
+ private final PersistingDiff parent;
+
+ private final String nodeName;
+
+ @Nonnull
+ private MemoryNodeBuilder builder;
+
+ @Nonnull
+ private final NodeState base;
+
+ @CheckForNull
+ private IOException exception;
+
+ private long modCount;
+
+ private PersistingDiff(PersistingDiff parent, String nodeName, @Nonnull
NodeState base) {
+ this.writer = parent.writer;
+ this.reader = parent.reader;
+ this.blobStore = parent.blobStore;
+ this.builder = new MemoryNodeBuilder(checkNotNull(base));
+ this.parent = parent;
+ this.base = base;
+ this.nodeName = nodeName;
+ }
+
+ private PersistingDiff(SegmentWriter writer, SegmentReader reader,
BlobStore blobStore, @Nonnull NodeState base) {
+ this.writer = writer;
+ this.reader = reader;
+ this.blobStore = blobStore;
+ this.builder = new MemoryNodeBuilder(checkNotNull(base));
+ this.parent = null;
+ this.base = base;
+ this.nodeName = null;
+ }
+
+ public static SegmentNodeState applyDiffOnNodeState(
+ FileStore fileStore,
+ @Nonnull NodeState before,
+ @Nonnull NodeState after,
+ @Nonnull NodeState onto) throws IOException {
+ return new PersistingDiff(fileStore.getWriter(),
fileStore.getReader(), fileStore.getBlobStore(), onto).diff(before, after);
+ }
+
+ private void updated() throws IOException {
+ if (modCount % UPDATE_LIMIT == 0) {
+ SegmentNodeState newBase =
writer.writeNode(builder.getNodeState());
+ builder = new MemoryNodeBuilder(newBase);
+ }
+ modCount++;
+ }
+
+ private String getPath() {
+ List<String> segments = new ArrayList<String>();
+ PersistingDiff currentDiff = this;
+ while (currentDiff != null) {
+ if (currentDiff.nodeName != null) {
+ segments.add(currentDiff.nodeName);
+ }
+ currentDiff = currentDiff.parent;
+ }
+ segments = Lists.reverse(segments);
+
+ StringBuilder path = new StringBuilder();
+ for (String segment : segments) {
+ path.append("/");
+ path.append(segment);
+ }
+ return path.toString();
+ }
+
+ @CheckForNull
+ SegmentNodeState diff(@Nonnull NodeState before, @Nonnull NodeState after)
throws IOException {
+ boolean success = after.compareAgainstBaseState(before, this);
+ if (exception != null) {
+ throw new IOException(exception);
+ } else if (success) {
+ NodeState nodeState = builder.getNodeState();
+ checkState(modCount == 0 || !(nodeState instanceof
SegmentNodeState));
+ return writer.writeNode(nodeState);
+ } else {
+ return null;
+ }
+ }
+
+ @Override
+ public boolean propertyAdded(@Nonnull PropertyState after) {
+ builder.setProperty(after);
+ return true;
+ }
+
+ @Override
+ public boolean propertyChanged(@Nonnull PropertyState before, @Nonnull
PropertyState after) {
+ builder.setProperty(after);
+ return true;
+ }
+
+ @Override
+ public boolean propertyDeleted(PropertyState before) {
+ builder.removeProperty(before.getName());
+ return true;
+ }
+
+ @Override
+ public boolean childNodeAdded(@Nonnull String name, @Nonnull NodeState
after) {
+ try {
+ SegmentNodeState segmentNodeState = new PersistingDiff(this, name,
EMPTY_NODE).diff(EMPTY_NODE, after);
+ if (segmentNodeState != null) {
+ updated();
+ builder.setChildNode(name, segmentNodeState);
+ return true;
+ } else {
+ return false;
+ }
+ } catch (IOException e) {
+ exception = e;
+ return false;
+ }
+ }
+
+ @Override
+ public boolean childNodeChanged(@Nonnull String name, @Nonnull NodeState
before, @Nonnull NodeState after) {
+ try {
+ SegmentNodeState compacted = new PersistingDiff(this, name,
base.getChildNode(name)).diff(before, after);
+ if (compacted != null) {
+ updated();
+ builder.setChildNode(name, compacted);
+ return true;
+ } else {
+ return false;
+ }
+ } catch (IOException e) {
+ exception = e;
+ return false;
+ }
+ }
+
+ @Override
+ public boolean childNodeDeleted(String name, NodeState before) {
+ try {
+ updated();
+ builder.getChildNode(name).remove();
+ return true;
+ } catch (IOException e) {
+ exception = e;
+ return false;
+ }
+ }
+}
Modified:
jackrabbit/oak/branches/1.6/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/RepositorySidegrade.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.6/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/RepositorySidegrade.java?rev=1815885&r1=1815884&r2=1815885&view=diff
==============================================================================
---
jackrabbit/oak/branches/1.6/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/RepositorySidegrade.java
(original)
+++
jackrabbit/oak/branches/1.6/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/RepositorySidegrade.java
Tue Nov 21 08:33:10 2017
@@ -16,35 +16,45 @@
*/
package org.apache.jackrabbit.oak.upgrade;
+import java.io.IOException;
import java.util.ArrayList;
import java.util.Calendar;
-import java.util.Comparator;
+import java.util.LinkedHashMap;
import java.util.List;
+import java.util.Map;
import java.util.Set;
import javax.annotation.Nonnull;
-import javax.annotation.Nullable;
import javax.jcr.RepositoryException;
-import com.google.common.base.Function;
+import com.google.common.collect.Lists;
import org.apache.commons.lang.StringUtils;
+import org.apache.jackrabbit.JcrConstants;
import org.apache.jackrabbit.oak.api.CommitFailedException;
+import org.apache.jackrabbit.oak.api.PropertyState;
+import org.apache.jackrabbit.oak.api.Type;
+import org.apache.jackrabbit.oak.commons.PathUtils;
+import org.apache.jackrabbit.oak.plugins.nodetype.TypePredicate;
+import org.apache.jackrabbit.oak.segment.SegmentNodeState;
+import org.apache.jackrabbit.oak.segment.file.FileStore;
import org.apache.jackrabbit.oak.spi.commit.CommitHook;
import org.apache.jackrabbit.oak.spi.commit.CommitInfo;
import org.apache.jackrabbit.oak.spi.commit.CompositeEditorProvider;
import org.apache.jackrabbit.oak.spi.commit.EditorHook;
-import org.apache.jackrabbit.oak.spi.lifecycle.RepositoryInitializer;
+import org.apache.jackrabbit.oak.spi.commit.EmptyHook;
import org.apache.jackrabbit.oak.spi.state.ApplyDiff;
import org.apache.jackrabbit.oak.spi.state.ChildNodeEntry;
import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
import org.apache.jackrabbit.oak.spi.state.NodeState;
import org.apache.jackrabbit.oak.spi.state.NodeStore;
import
org.apache.jackrabbit.oak.upgrade.RepositoryUpgrade.LoggingCompositeHook;
-import org.apache.jackrabbit.oak.upgrade.cli.node.TarNodeStore;
+import org.apache.jackrabbit.oak.upgrade.checkpoint.CheckpointRetriever;
+import org.apache.jackrabbit.oak.upgrade.cli.node.SegmentTarFactory;
+import org.apache.jackrabbit.oak.upgrade.nodestate.FilteringNodeState;
import org.apache.jackrabbit.oak.upgrade.nodestate.NameFilteringNodeState;
+import org.apache.jackrabbit.oak.upgrade.nodestate.NodeStateCopier;
import org.apache.jackrabbit.oak.upgrade.nodestate.report.LoggingReporter;
import org.apache.jackrabbit.oak.upgrade.nodestate.report.ReportingNodeState;
-import org.apache.jackrabbit.oak.upgrade.nodestate.NodeStateCopier;
import org.apache.jackrabbit.oak.upgrade.version.VersionCopyConfiguration;
import org.apache.jackrabbit.oak.upgrade.version.VersionHistoryUtil;
import org.apache.jackrabbit.oak.upgrade.version.VersionableEditor;
@@ -54,21 +64,19 @@ import org.slf4j.LoggerFactory;
import static com.google.common.base.Preconditions.checkNotNull;
import static com.google.common.collect.ImmutableSet.copyOf;
import static com.google.common.collect.ImmutableSet.of;
-import static com.google.common.collect.Lists.newArrayList;
-import static com.google.common.collect.Lists.transform;
import static com.google.common.collect.Sets.union;
-import static java.util.Collections.sort;
import static org.apache.jackrabbit.JcrConstants.JCR_PRIMARYTYPE;
import static org.apache.jackrabbit.JcrConstants.JCR_SYSTEM;
import static
org.apache.jackrabbit.oak.spi.security.authorization.permission.PermissionConstants.NT_REP_PERMISSION_STORE;
import static
org.apache.jackrabbit.oak.spi.security.authorization.permission.PermissionConstants.REP_PERMISSION_STORE;
+import static
org.apache.jackrabbit.oak.upgrade.RepositoryUpgrade.DEFAULT_EXCLUDE_FRAGMENTS;
import static
org.apache.jackrabbit.oak.upgrade.RepositoryUpgrade.DEFAULT_EXCLUDE_PATHS;
+import static
org.apache.jackrabbit.oak.upgrade.RepositoryUpgrade.DEFAULT_FRAGMENT_PATHS;
import static
org.apache.jackrabbit.oak.upgrade.RepositoryUpgrade.DEFAULT_INCLUDE_PATHS;
import static
org.apache.jackrabbit.oak.upgrade.RepositoryUpgrade.DEFAULT_MERGE_PATHS;
import static
org.apache.jackrabbit.oak.upgrade.RepositoryUpgrade.calculateEffectiveIncludePaths;
import static
org.apache.jackrabbit.oak.upgrade.RepositoryUpgrade.createIndexEditorProvider;
import static
org.apache.jackrabbit.oak.upgrade.RepositoryUpgrade.createTypeEditorProvider;
-import static
org.apache.jackrabbit.oak.upgrade.RepositoryUpgrade.markIndexesToBeRebuilt;
import static
org.apache.jackrabbit.oak.upgrade.nodestate.NodeStateCopier.copyProperties;
import static
org.apache.jackrabbit.oak.upgrade.version.VersionCopier.copyVersionStorage;
import static
org.apache.jackrabbit.oak.upgrade.version.VersionHistoryUtil.getVersionStorage;
@@ -77,7 +85,7 @@ public class RepositorySidegrade {
private static final Logger LOG =
LoggerFactory.getLogger(RepositorySidegrade.class);
- private static final int LOG_NODE_COPY =
Integer.getInteger("oak.upgrade.logNodeCopy", 10000);
+ static final int LOG_NODE_COPY =
Integer.getInteger("oak.upgrade.logNodeCopy", 10000);
private static final String WORKSPACE_NAME_PROP =
"oak.upgrade.workspaceName";
@@ -86,6 +94,8 @@ public class RepositorySidegrade {
*/
private final NodeStore target;
+ private final FileStore targetFileStore;
+
private final NodeStore source;
/**
@@ -99,12 +109,24 @@ public class RepositorySidegrade {
private Set<String> excludePaths = DEFAULT_EXCLUDE_PATHS;
/**
+ * Paths supporting fragments during the copy process. Empty by default.
+ */
+ private Set<String> fragmentPaths = DEFAULT_FRAGMENT_PATHS;
+
+ /**
+ * Fragments to exclude during the copy process. Empty by default.
+ */
+ private Set<String> excludeFragments = DEFAULT_EXCLUDE_FRAGMENTS;
+
+ /**
* Paths to merge during the copy process. Empty by default.
*/
private Set<String> mergePaths = DEFAULT_MERGE_PATHS;
private boolean skipCheckpoints = false;
+ private boolean forceCheckpoints = false;
+
private boolean includeIndex = false;
private boolean filterLongNames = true;
@@ -163,6 +185,12 @@ public class RepositorySidegrade {
public RepositorySidegrade(NodeStore source, NodeStore target) {
this.source = source;
this.target = target;
+
+ FileStore fs = null;
+ if (target instanceof SegmentTarFactory.NodeStoreWithFileStore) {
+ fs = ((SegmentTarFactory.NodeStoreWithFileStore)
target).getFileStore();
+ }
+ this.targetFileStore = fs;
}
/**
@@ -205,6 +233,25 @@ public class RepositorySidegrade {
this.excludePaths = copyOf(checkNotNull(excludes));
}
+ /**
+ * Sets the paths that should support the fragments.
+ *
+ * @param fragmentPaths Paths that should support fragments.
+ */
+ public void setFragmentPaths(@Nonnull String... fragmentPaths) {
+ this.fragmentPaths = copyOf(checkNotNull(fragmentPaths));
+ }
+
+ /**
+ * Sets the name fragments that should be excluded when the source
repository
+ * is copied to the target repository.
+ *
+ * @param excludes Name fragments to be excluded from the copy.
+ */
+ public void setExcludeFragments(@Nonnull String... excludes) {
+ this.excludeFragments = copyOf(checkNotNull(excludes));
+ }
+
public void setIncludeIndex(boolean includeIndex) {
this.includeIndex = includeIndex;
}
@@ -219,10 +266,6 @@ public class RepositorySidegrade {
this.mergePaths = copyOf(checkNotNull(merges));
}
- public boolean isFilterLongNames() {
- return filterLongNames;
- }
-
public void setFilterLongNames(boolean filterLongNames) {
this.filterLongNames = filterLongNames;
}
@@ -239,13 +282,8 @@ public class RepositorySidegrade {
this.skipCheckpoints = skipCheckpoints;
}
- /**
- * Same as {@link #copy(RepositoryInitializer)}, but with no custom
initializer.
- *
- * @throws RepositoryException if the copy operation fails
- */
- public void copy() throws RepositoryException {
- copy(null);
+ public void setForceCheckpoints(boolean forceCheckpoints) {
+ this.forceCheckpoints = forceCheckpoints;
}
/**
@@ -257,32 +295,17 @@ public class RepositorySidegrade {
* Note that both the source and the target repository must be closed
* during the copy operation as this method requires exclusive access
* to the repositories.
- *
- * @param initializer optional repository initializer to use
*
* @throws RepositoryException if the copy operation fails
*/
- public void copy(RepositoryInitializer initializer) throws
RepositoryException {
+ public void copy() throws RepositoryException {
try {
if (!onlyVerify) {
- NodeBuilder targetRoot = target.getRoot().builder();
- if (VersionHistoryUtil.getVersionStorage(targetRoot).exists()
&& !versionCopyConfiguration.skipOrphanedVersionsCopy()) {
+ if
(VersionHistoryUtil.getVersionStorage(target.getRoot()).exists() &&
!versionCopyConfiguration.skipOrphanedVersionsCopy()) {
LOG.warn("The version storage on destination already
exists. Orphaned version histories will be skipped.");
versionCopyConfiguration.setCopyOrphanedVersions(null);
}
-
- if (initializer != null) {
- initializer.initialize(targetRoot);
- }
-
- final NodeState reportingSourceRoot =
ReportingNodeState.wrap(source.getRoot(), new LoggingReporter(LOG, "Copying",
LOG_NODE_COPY, -1));
- final NodeState sourceRoot;
- if (filterLongNames) {
- sourceRoot =
NameFilteringNodeState.wrap(reportingSourceRoot);
- } else {
- sourceRoot = reportingSourceRoot;
- }
- copyState(sourceRoot, targetRoot);
+ copyState();
}
if (verify || onlyVerify) {
verify();
@@ -299,42 +322,137 @@ public class RepositorySidegrade {
builder.setChildNode(":async");
}
- private void copyState(NodeState sourceRoot, NodeBuilder targetRoot)
throws CommitFailedException, RepositoryException {
- copyWorkspace(sourceRoot, targetRoot);
+ private void copyState() throws CommitFailedException,
RepositoryException, IOException {
+ boolean migrateCheckpoints = true;
+ if (!isCompleteMigration() && !forceCheckpoints) {
+ LOG.info("Checkpoints won't be migrated because of the specified
paths");
+ migrateCheckpoints = false;
+ }
+ if (!versionCopyConfiguration.isCopyAll()) {
+ LOG.info("Checkpoints won't be migrated because of the specified
version settings");
+ migrateCheckpoints = false;
+ }
+ if (skipCheckpoints) {
+ LOG.info("Checkpoints won't be migrated because of the
--skip-checkpoints options");
+ migrateCheckpoints = false;
+ }
+ if (targetExists()) {
+ LOG.info("Checkpoints won't be migrated because the destination
repository exists");
+ migrateCheckpoints = false;
+ }
+ if (migrateCheckpoints) {
+ migrateCheckpoints = migrateWithCheckpoints();
+ }
+ if (!migrateCheckpoints) {
+ migrateWithoutCheckpoints();
+ }
+ }
- if (includeIndex) {
- IndexCopier.copy(sourceRoot, targetRoot, includePaths);
+ private boolean migrateWithCheckpoints() throws CommitFailedException,
IOException {
+ List<CheckpointRetriever.Checkpoint> checkpoints =
CheckpointRetriever.getCheckpoints(source);
+ if (checkpoints == null) {
+ return false;
}
- boolean isRemoveCheckpointReferences = false;
- if (!isCompleteMigration()) {
- LOG.info("Custom paths have been specified, checkpoints won't be
migrated");
- isRemoveCheckpointReferences = true;
- } else if (skipCheckpoints) {
- LOG.info("Checkpoints won't be migrated because of the
--skip-checkpoints option");
- isRemoveCheckpointReferences = true;
+ Map<String, String> nameToRevision = new LinkedHashMap<>();
+ Map<String, String> checkpointSegmentToDoc = new LinkedHashMap<>();
+
+ NodeState initialRoot = target.getRoot();
+ NodeState targetRoot = initialRoot;
+ NodeState previousRoot = initialRoot;
+ for (CheckpointRetriever.Checkpoint checkpoint : checkpoints) {
+ NodeState checkpointRoot = source.retrieve(checkpoint.getName());
+ Map<String, String> checkpointInfo =
source.checkpointInfo(checkpoint.getName());
+
+ boolean tracePaths;
+ if (previousRoot == initialRoot) {
+ LOG.info("Migrating first checkpoint: {}",
checkpoint.getName());
+ tracePaths = true;
+ } else {
+ LOG.info("Applying diff to {}", checkpoint.getName());
+ tracePaths = false;
+ }
+ LOG.info("Checkpoint expiry time: {}, metadata: {}",
checkpoint.getExpiryTime(), checkpointInfo);
+
+ targetRoot = copyDiffToTarget(previousRoot, checkpointRoot,
targetRoot, tracePaths);
+ previousRoot = checkpointRoot;
+
+ String newCheckpointName =
target.checkpoint(checkpoint.getExpiryTime() - System.currentTimeMillis(),
checkpointInfo);
+ if (checkpointInfo.containsKey("name")) {
+ nameToRevision.put(checkpointInfo.get("name"),
newCheckpointName);
+ }
+ checkpointSegmentToDoc.put(checkpoint.getName(),
newCheckpointName);
+ }
+
+ NodeState sourceRoot = source.getRoot();
+ boolean tracePaths;
+ if (previousRoot == initialRoot) {
+ LOG.info("No checkpoints found; migrating head");
+ tracePaths = true;
} else {
- boolean checkpointsCopied;
- try {
- checkpointsCopied = copyCheckpoints(targetRoot);
- } catch(UnsupportedOperationException e) {
- removeCheckpoints();
- throw new RepositoryException("Checkpoints won't be copied,
because no external datastore has been specified. This will result in the full
repository reindexing on the first start. Use --skip-checkpoints to force the
migration or see
https://jackrabbit.apache.org/oak/docs/migration.html#Checkpoints_migration for
more info.");
+ LOG.info("Applying diff to head");
+ tracePaths = false;
+ }
+
+ targetRoot = copyDiffToTarget(previousRoot, sourceRoot, targetRoot,
tracePaths);
+
+ LOG.info("Rewriting checkpoint names in /:async {}", nameToRevision);
+ NodeBuilder targetBuilder = targetRoot.builder();
+ NodeBuilder async = targetBuilder.getChildNode(":async");
+ for (Map.Entry<String, String> e : nameToRevision.entrySet()) {
+ async.setProperty(e.getKey(), e.getValue(), Type.STRING);
+
+ PropertyState temp = async.getProperty(e.getKey() + "-temp");
+ if (temp == null) {
+ continue;
}
- if (!checkpointsCopied) {
- LOG.info("Copying checkpoints is not supported for this
combination of node stores");
- isRemoveCheckpointReferences = true;
+ List<String> tempValues =
Lists.newArrayList(temp.getValue(Type.STRINGS));
+ for (Map.Entry<String, String> sToD :
checkpointSegmentToDoc.entrySet()) {
+ if (tempValues.contains(sToD.getKey())) {
+ tempValues.set(tempValues.indexOf(sToD.getKey()),
sToD.getValue());
+ }
}
+ async.setProperty(e.getKey() + "-temp", tempValues, Type.STRINGS);
+ }
+ target.merge(targetBuilder, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+ return true;
+ }
+
+ private NodeState copyDiffToTarget(NodeState before, NodeState after,
NodeState targetRoot, boolean tracePaths) throws IOException,
CommitFailedException {
+
+ NodeBuilder targetBuilder = targetRoot.builder();
+ if (targetFileStore == null) {
+ NodeState currentRoot = wrapNodeState(after, tracePaths, true);
+ NodeState baseRoot = wrapNodeState(before, false, true);
+
+ currentRoot.compareAgainstBaseState(baseRoot, new
ApplyDiff(targetBuilder));
+ } else {
+ NodeState currentRoot = wrapNodeState(after, false, true);
+ NodeState baseRoot = wrapNodeState(before, false, true);
+
+ SegmentNodeState state =
PersistingDiff.applyDiffOnNodeState(targetFileStore, baseRoot, currentRoot,
targetRoot);
+ state.compareAgainstBaseState(targetRoot, new
ApplyDiff(targetBuilder));
}
- if (isRemoveCheckpointReferences) {
- removeCheckpointReferences(targetRoot);
+ return target.merge(targetBuilder, EmptyHook.INSTANCE,
CommitInfo.EMPTY);
+ }
+
+ private void migrateWithoutCheckpoints() throws CommitFailedException,
RepositoryException {
+ final List<CommitHook> hooks = new ArrayList<>();
+ if (customCommitHooks != null) {
+ hooks.addAll(customCommitHooks);
}
- final List<CommitHook> hooks = new ArrayList<CommitHook>();
+ NodeState sourceRoot = wrapNodeState(source.getRoot(), true, false);
+ NodeBuilder targetRoot = target.getRoot().builder();
+ copyWorkspace(sourceRoot, targetRoot);
+ removeCheckpointReferences(targetRoot);
+ if (includeIndex) {
+ IndexCopier.copy(sourceRoot, targetRoot, includePaths);
+ }
if (!versionCopyConfiguration.isCopyAll()) {
NodeBuilder versionStorage =
VersionHistoryUtil.getVersionStorage(targetRoot);
if (!versionStorage.exists()) { // it's possible that this is a
new repository and the version storage
- // hasn't been created/copied yet
+ // hasn't been created/copied yet
versionStorage =
VersionHistoryUtil.createVersionStorage(targetRoot);
}
if (!versionCopyConfiguration.skipOrphanedVersionsCopy()) {
@@ -342,25 +460,28 @@ public class RepositorySidegrade {
}
hooks.add(new EditorHook(new
VersionableEditor.Provider(sourceRoot, getWorkspaceName(),
versionCopyConfiguration)));
}
+ // type validation, reference and indexing hooks
+ hooks.add(new EditorHook(new CompositeEditorProvider(
+ createTypeEditorProvider(),
+ createIndexEditorProvider()
+ )));
+ target.merge(targetRoot, new LoggingCompositeHook(hooks, null, false),
CommitInfo.EMPTY);
+ removeVersions();
+ }
- if (customCommitHooks != null) {
- hooks.addAll(customCommitHooks);
- }
-
- if (!isCompleteMigration()) {
- markIndexesToBeRebuilt(targetRoot);
- // type validation, reference and indexing hooks
- hooks.add(new EditorHook(new CompositeEditorProvider(
- createTypeEditorProvider(),
- createIndexEditorProvider()
- )));
+ private void removeVersions() throws CommitFailedException {
+ NodeState root = target.getRoot();
+ NodeState wrappedRoot = FilteringNodeState.wrap(PathUtils.ROOT_PATH,
root, includePaths, excludePaths, fragmentPaths, excludeFragments);
+ List<String> versionablesToStrip =
VersionHistoryUtil.getVersionableNodes(wrappedRoot, new TypePredicate(root,
JcrConstants.MIX_VERSIONABLE), versionCopyConfiguration.getVersionsMinDate());
+ if (!versionablesToStrip.isEmpty()) {
+ LOG.info("Removing version histories for included paths");
+ NodeBuilder newRoot = VersionHistoryUtil.removeVersions(root,
versionablesToStrip);
+ target.merge(newRoot, EmptyHook.INSTANCE, CommitInfo.EMPTY);
}
-
- target.merge(targetRoot, new LoggingCompositeHook(hooks, null, false),
CommitInfo.EMPTY);
}
private boolean isCompleteMigration() {
- return includePaths.equals(DEFAULT_INCLUDE_PATHS) &&
excludePaths.equals(DEFAULT_EXCLUDE_PATHS) &&
mergePaths.equals(DEFAULT_MERGE_PATHS);
+ return includePaths.equals(DEFAULT_INCLUDE_PATHS) &&
excludePaths.equals(DEFAULT_EXCLUDE_PATHS) &&
excludeFragments.equals(DEFAULT_EXCLUDE_FRAGMENTS) &&
mergePaths.equals(DEFAULT_MERGE_PATHS) &&
fragmentPaths.equals(DEFAULT_FRAGMENT_PATHS);
}
private void copyWorkspace(NodeState sourceRoot, NodeBuilder targetRoot) {
@@ -376,6 +497,8 @@ public class RepositorySidegrade {
NodeStateCopier.builder()
.include(includes)
.exclude(excludes)
+ .supportFragment(fragmentPaths)
+ .excludeFragments(excludeFragments)
.merge(merges)
.copy(sourceRoot, targetRoot);
@@ -384,93 +507,6 @@ public class RepositorySidegrade {
}
}
- private boolean copyCheckpoints(NodeBuilder targetRoot) {
- if (!(source instanceof TarNodeStore && target instanceof
TarNodeStore)) {
- return false;
- }
-
- TarNodeStore sourceTarNS = (TarNodeStore) source;
- TarNodeStore targetTarNS = (TarNodeStore) target;
-
- NodeState sourceSuperRoot = sourceTarNS.getSuperRoot();
- NodeBuilder targetSuperRoot = targetTarNS.getSuperRoot().builder();
-
- String previousCheckpoint = null;
- for (String checkpoint : getCheckpointNames(sourceSuperRoot)) {
- NodeState targetPreviousRoot, sourcePreviousRoot;
- if (previousCheckpoint == null) {
- sourcePreviousRoot = source.getRoot();
- targetPreviousRoot = targetRoot.getNodeState();
- } else {
- sourcePreviousRoot = getCheckpointRoot(sourceSuperRoot,
previousCheckpoint);
- targetPreviousRoot =
getCheckpointRoot(targetSuperRoot.getNodeState(), previousCheckpoint);
- }
- NodeState sourceCheckpoint = getCheckpoint(sourceSuperRoot,
checkpoint);
- NodeBuilder targetCheckpoint = getCheckpoint(targetSuperRoot,
checkpoint);
-
- // copy checkpoint metadata
- NodeStateCopier.copyProperties(sourceCheckpoint, targetCheckpoint);
- targetCheckpoint.setChildNode("properties",
sourceCheckpoint.getChildNode("properties"));
-
- // create the checkpoint root
- NodeState sourceCheckpointRoot =
sourceCheckpoint.getChildNode("root");
- NodeBuilder targetCheckpointRoot =
targetCheckpoint.setChildNode("root", targetPreviousRoot);
- sourceCheckpointRoot.compareAgainstBaseState(sourcePreviousRoot,
new ApplyDiff(targetCheckpointRoot));
-
- previousCheckpoint = checkpoint;
- }
-
- targetTarNS.setSuperRoot(targetSuperRoot);
- return true;
- }
-
- private void removeCheckpoints() {
- if (!(target instanceof TarNodeStore)) {
- return;
- }
- TarNodeStore targetTarNS = (TarNodeStore) target;
- NodeBuilder targetSuperRoot = ((TarNodeStore)
target).getSuperRoot().builder();
- targetSuperRoot.setChildNode("checkpoints");
- targetTarNS.setSuperRoot(targetSuperRoot);
- }
-
- /**
- * Return all checkpoint paths, sorted by their "created" property,
descending.
- *
- * @param superRoot
- * @return
- */
- private static List<String> getCheckpointNames(NodeState superRoot) {
- List<ChildNodeEntry> checkpoints =
newArrayList(superRoot.getChildNode("checkpoints").getChildNodeEntries().iterator());
- sort(checkpoints, new Comparator<ChildNodeEntry>() {
- @Override
- public int compare(ChildNodeEntry o1, ChildNodeEntry o2) {
- long c1 = o1.getNodeState().getLong("created");
- long c2 = o1.getNodeState().getLong("created");
- return -Long.compare(c1, c2);
- }
- });
- return transform(checkpoints, new Function<ChildNodeEntry, String>() {
- @Nullable
- @Override
- public String apply(@Nullable ChildNodeEntry input) {
- return input.getName();
- }
- });
- }
-
- private static NodeState getCheckpointRoot(NodeState superRoot, String
name) {
- return getCheckpoint(superRoot, name).getChildNode("root");
- }
-
- private static NodeState getCheckpoint(NodeState superRoot, String name) {
- return superRoot.getChildNode("checkpoints").getChildNode(name);
- }
-
- private static NodeBuilder getCheckpoint(NodeBuilder superRoot, String
name) {
- return superRoot.child("checkpoints").child(name);
- }
-
private String getWorkspaceName() throws RepositoryException {
String definedName = System.getProperty(WORKSPACE_NAME_PROP);
String detectedName = deriveWorkspaceName();
@@ -507,16 +543,8 @@ public class RepositorySidegrade {
}
private void verify() {
- final NodeState sourceRoot;
- final NodeState targetRoot;
-
- if (source instanceof TarNodeStore && target instanceof TarNodeStore) {
- sourceRoot = ((TarNodeStore) source).getSuperRoot();
- targetRoot = ((TarNodeStore) target).getSuperRoot();
- } else {
- sourceRoot = source.getRoot();
- targetRoot = target.getRoot();
- }
+ final NodeState sourceRoot = source.getRoot();
+ final NodeState targetRoot = target.getRoot();
final NodeState reportingSource = ReportingNodeState.wrap(sourceRoot,
new LoggingReporter(LOG, "Verifying", LOG_NODE_COPY, -1));
@@ -527,4 +555,23 @@ public class RepositorySidegrade {
LOG.warn("Verification result: repositories are not identical");
}
}
+
+ private NodeState wrapNodeState(NodeState source, boolean tracePaths,
boolean filterPaths) {
+ NodeState wrapped = source;
+ if (!isCompleteMigration() && filterPaths) {
+ wrapped = FilteringNodeState.wrap("/", wrapped, includePaths,
excludePaths, fragmentPaths, excludeFragments);
+ }
+ if (tracePaths) {
+ wrapped = ReportingNodeState.wrap(wrapped, new
LoggingReporter(LOG, "Copying", LOG_NODE_COPY, -1));
+ }
+ if (filterLongNames) {
+ wrapped = NameFilteringNodeState.wrap(wrapped);
+ }
+ return wrapped;
+ }
+
+ private boolean targetExists() {
+ return target.getRoot().getChildNodeEntries().iterator().hasNext();
+ }
+
}
Modified:
jackrabbit/oak/branches/1.6/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/RepositoryUpgrade.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.6/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/RepositoryUpgrade.java?rev=1815885&r1=1815884&r2=1815885&view=diff
==============================================================================
---
jackrabbit/oak/branches/1.6/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/RepositoryUpgrade.java
(original)
+++
jackrabbit/oak/branches/1.6/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/RepositoryUpgrade.java
Tue Nov 21 08:33:10 2017
@@ -25,23 +25,17 @@ import static com.google.common.collect.
import static com.google.common.collect.Maps.newHashMap;
import static com.google.common.collect.Sets.newHashSet;
import static com.google.common.collect.Sets.union;
-import static java.util.Collections.emptyMap;
import static org.apache.jackrabbit.JcrConstants.JCR_SYSTEM;
-import static
org.apache.jackrabbit.oak.plugins.index.IndexConstants.REINDEX_PROPERTY_NAME;
import static
org.apache.jackrabbit.oak.plugins.name.Namespaces.addCustomMapping;
import static
org.apache.jackrabbit.oak.plugins.nodetype.NodeTypeConstants.NODE_TYPES_PATH;
import static
org.apache.jackrabbit.oak.spi.security.privilege.PrivilegeConstants.JCR_ALL;
import static
org.apache.jackrabbit.oak.upgrade.cli.parser.OptionParserFactory.SKIP_NAME_CHECK;
-import static
org.apache.jackrabbit.oak.upgrade.nodestate.FilteringNodeState.ALL;
-import static
org.apache.jackrabbit.oak.upgrade.nodestate.FilteringNodeState.NONE;
-import static
org.apache.jackrabbit.oak.upgrade.nodestate.NodeStateCopier.copyProperties;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.util.Calendar;
import java.util.Collection;
-import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
@@ -68,8 +62,8 @@ import com.google.common.base.Stopwatch;
import com.google.common.collect.HashBiMap;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Lists;
+import org.apache.jackrabbit.JcrConstants;
import org.apache.jackrabbit.api.security.authorization.PrivilegeManager;
import org.apache.jackrabbit.core.IndexAccessor;
import org.apache.jackrabbit.core.RepositoryContext;
@@ -84,24 +78,24 @@ import org.apache.jackrabbit.core.query.
import org.apache.jackrabbit.core.security.authorization.PrivilegeRegistry;
import org.apache.jackrabbit.core.security.user.UserManagerImpl;
import org.apache.jackrabbit.oak.api.CommitFailedException;
-import org.apache.jackrabbit.oak.api.PropertyState;
import org.apache.jackrabbit.oak.api.Root;
import org.apache.jackrabbit.oak.api.Tree;
-import org.apache.jackrabbit.oak.api.Type;
+import org.apache.jackrabbit.oak.commons.PathUtils;
import org.apache.jackrabbit.oak.namepath.NamePathMapper;
import org.apache.jackrabbit.oak.plugins.index.CompositeIndexEditorProvider;
import org.apache.jackrabbit.oak.plugins.index.IndexEditorProvider;
import org.apache.jackrabbit.oak.plugins.index.IndexUpdate;
import org.apache.jackrabbit.oak.plugins.index.IndexUpdateCallback;
-import org.apache.jackrabbit.oak.plugins.index.IndexUtils;
import
org.apache.jackrabbit.oak.plugins.index.property.PropertyIndexEditorProvider;
import
org.apache.jackrabbit.oak.plugins.index.reference.ReferenceEditorProvider;
import org.apache.jackrabbit.oak.plugins.name.NamespaceConstants;
+import org.apache.jackrabbit.oak.plugins.nodetype.TypePredicate;
+import org.apache.jackrabbit.oak.plugins.nodetype.write.InitialContent;
+import org.apache.jackrabbit.oak.plugins.value.ValueFactoryImpl;
+import org.apache.jackrabbit.oak.spi.commit.EmptyHook;
import org.apache.jackrabbit.oak.plugins.name.Namespaces;
import org.apache.jackrabbit.oak.plugins.nodetype.TypeEditorProvider;
-import org.apache.jackrabbit.oak.plugins.nodetype.write.InitialContent;
import
org.apache.jackrabbit.oak.plugins.nodetype.write.ReadWriteNodeTypeManager;
-import org.apache.jackrabbit.oak.plugins.value.ValueFactoryImpl;
import org.apache.jackrabbit.oak.security.SecurityProviderImpl;
import org.apache.jackrabbit.oak.spi.commit.CommitHook;
import org.apache.jackrabbit.oak.spi.commit.CommitInfo;
@@ -121,10 +115,11 @@ import org.apache.jackrabbit.oak.spi.sec
import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
import org.apache.jackrabbit.oak.spi.state.NodeState;
import org.apache.jackrabbit.oak.spi.state.NodeStore;
+import org.apache.jackrabbit.oak.upgrade.nodestate.FilteringNodeState;
import org.apache.jackrabbit.oak.upgrade.nodestate.NameFilteringNodeState;
+import org.apache.jackrabbit.oak.upgrade.nodestate.NodeStateCopier;
import org.apache.jackrabbit.oak.upgrade.nodestate.report.LoggingReporter;
import org.apache.jackrabbit.oak.upgrade.nodestate.report.ReportingNodeState;
-import org.apache.jackrabbit.oak.upgrade.nodestate.NodeStateCopier;
import org.apache.jackrabbit.oak.upgrade.security.AuthorizableFolderEditor;
import org.apache.jackrabbit.oak.upgrade.security.GroupEditorProvider;
import org.apache.jackrabbit.oak.upgrade.security.RestrictionEditorProvider;
@@ -148,6 +143,9 @@ import org.apache.lucene.index.TermEnum;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import static
org.apache.jackrabbit.oak.upgrade.nodestate.FilteringNodeState.ALL;
+import static
org.apache.jackrabbit.oak.upgrade.nodestate.FilteringNodeState.NONE;
+import static
org.apache.jackrabbit.oak.upgrade.nodestate.NodeStateCopier.copyProperties;
import static
org.apache.jackrabbit.oak.upgrade.version.VersionCopier.copyVersionStorage;
import static
org.apache.jackrabbit.oak.upgrade.version.VersionHistoryUtil.getVersionStorage;
@@ -157,12 +155,14 @@ public class RepositoryUpgrade {
private static final int LOG_NODE_COPY =
Integer.getInteger("oak.upgrade.logNodeCopy", 10000);
- private static final Set<String> INDEXES_TO_REBUILD =
ImmutableSet.of("counter", "uuid");
-
public static final Set<String> DEFAULT_INCLUDE_PATHS = ALL;
public static final Set<String> DEFAULT_EXCLUDE_PATHS = NONE;
+ public static final Set<String> DEFAULT_FRAGMENT_PATHS = NONE;
+
+ public static final Set<String> DEFAULT_EXCLUDE_FRAGMENTS = NONE;
+
public static final Set<String> DEFAULT_MERGE_PATHS = NONE;
/**
@@ -186,6 +186,16 @@ public class RepositoryUpgrade {
private Set<String> excludePaths = DEFAULT_EXCLUDE_PATHS;
/**
+ * Paths supporting fragments during the copy process. Empty by default.
+ */
+ private Set<String> fragmentPaths = DEFAULT_FRAGMENT_PATHS;
+
+ /**
+ * Fragments to exclude during the copy process. Empty by default.
+ */
+ private Set<String> excludeFragments = DEFAULT_EXCLUDE_FRAGMENTS;
+
+ /**
* Paths to merge during the copy process. Empty by default.
*/
private Set<String> mergePaths = DEFAULT_MERGE_PATHS;
@@ -342,6 +352,25 @@ public class RepositoryUpgrade {
}
/**
+ * Sets the paths that should support the fragments.
+ *
+ * @param fragmentPaths Paths that should support fragments.
+ */
+ public void setFragmentPaths(@Nonnull String... fragmentPaths) {
+ this.fragmentPaths = copyOf(checkNotNull(fragmentPaths));
+ }
+
+ /**
+ * Sets the name fragments that should be excluded when the source
repository
+ * is copied to the target repository.
+ *
+ * @param excludes Name fragments to be excluded from the copy.
+ */
+ public void setExcludeFragments(@Nonnull String... excludes) {
+ this.excludeFragments = copyOf(checkNotNull(excludes));
+ }
+
+ /**
* Sets the paths that should be merged when the source repository
* is copied to the target repository.
*
@@ -546,8 +575,6 @@ public class RepositoryUpgrade {
hooks.addAll(customCommitHooks);
}
- markIndexesToBeRebuilt(targetBuilder);
-
// type validation, reference and indexing hooks
hooks.add(new EditorHook(new CompositeEditorProvider(
createTypeEditorProvider(),
@@ -556,24 +583,23 @@ public class RepositoryUpgrade {
target.merge(targetBuilder, new LoggingCompositeHook(hooks,
source, overrideEarlyShutdown()), CommitInfo.EMPTY);
logger.info("Processing commit hooks completed in {}s ({})",
watch.elapsed(TimeUnit.SECONDS), watch);
+
+ removeVersions();
+
logger.debug("Repository upgrade completed.");
} catch (Exception e) {
throw new RepositoryException("Failed to copy content", e);
}
}
- static void markIndexesToBeRebuilt(NodeBuilder targetRoot) {
- NodeBuilder oakIndex = IndexUtils.getOrCreateOakIndex(targetRoot);
- for (String indexName : INDEXES_TO_REBUILD) {
- final NodeBuilder indexDef = oakIndex.getChildNode(indexName);
- if (!indexDef.exists()) {
- continue;
- }
- final PropertyState reindex =
indexDef.getProperty(REINDEX_PROPERTY_NAME);
- logger.info("Marking {} to be reindexed", indexName);
- if (reindex == null || !reindex.getValue(Type.BOOLEAN)) {
- indexDef.setProperty(REINDEX_PROPERTY_NAME, true);
- }
+ private void removeVersions() throws CommitFailedException {
+ NodeState root = target.getRoot();
+ NodeState wrappedRoot = FilteringNodeState.wrap(PathUtils.ROOT_PATH,
root, includePaths, excludePaths, fragmentPaths, excludeFragments);
+ List<String> versionablesToStrip =
VersionHistoryUtil.getVersionableNodes(wrappedRoot, new TypePredicate(root,
JcrConstants.MIX_VERSIONABLE), versionCopyConfiguration.getVersionsMinDate());
+ if (!versionablesToStrip.isEmpty()) {
+ logger.info("Removing version histories for included paths");
+ NodeBuilder newRoot = VersionHistoryUtil.removeVersions(root,
versionablesToStrip);
+ target.merge(newRoot, EmptyHook.INSTANCE, CommitInfo.EMPTY);
}
}
@@ -592,6 +618,10 @@ public class RepositoryUpgrade {
logger.info("Overriding early shutdown to false because of the
copy versions settings");
return false;
}
+ if (c.isCopyVersions() &&
target.getRoot().hasChildNode(JcrConstants.JCR_SYSTEM)) {
+ logger.info("Overriding early shutdown to false because the target
exists");
+ return false;
+ }
return true;
}
@@ -957,6 +987,8 @@ public class RepositoryUpgrade {
NodeStateCopier.builder()
.include(includes)
.exclude(excludes)
+ .supportFragment(fragmentPaths)
+ .excludeFragments(excludeFragments)
.merge(merges)
.copy(sourceRoot, targetRoot);
@@ -1023,10 +1055,6 @@ public class RepositoryUpgrade {
private final boolean earlyShutdown;
private final RepositoryContext source;
- public LoggingCompositeHook(Collection<CommitHook> hooks) {
- this(hooks, null, false);
- }
-
public LoggingCompositeHook(Collection<CommitHook> hooks,
RepositoryContext source, boolean earlyShutdown) {
this.hooks = hooks;
Added:
jackrabbit/oak/branches/1.6/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/checkpoint/CheckpointRetriever.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.6/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/checkpoint/CheckpointRetriever.java?rev=1815885&view=auto
==============================================================================
---
jackrabbit/oak/branches/1.6/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/checkpoint/CheckpointRetriever.java
(added)
+++
jackrabbit/oak/branches/1.6/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/checkpoint/CheckpointRetriever.java
Tue Nov 21 08:33:10 2017
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.jackrabbit.oak.upgrade.checkpoint;
+
+import com.google.common.base.Function;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
+import org.apache.jackrabbit.oak.plugins.document.DocumentCheckpointRetriever;
+import org.apache.jackrabbit.oak.plugins.document.DocumentNodeStore;
+import org.apache.jackrabbit.oak.segment.CheckpointAccessor;
+import org.apache.jackrabbit.oak.segment.SegmentNodeStore;
+import org.apache.jackrabbit.oak.spi.state.ChildNodeEntry;
+import org.apache.jackrabbit.oak.spi.state.NodeState;
+import org.apache.jackrabbit.oak.spi.state.NodeStore;
+import org.apache.jackrabbit.oak.upgrade.cli.node.SegmentTarFactory;
+
+import javax.annotation.Nullable;
+import java.util.Collections;
+import java.util.List;
+
+public final class CheckpointRetriever {
+
+ public static class Checkpoint implements Comparable<Checkpoint> {
+
+ private final String name;
+
+ private final long created;
+
+ private final long expiryTime;
+
+ public Checkpoint(String name, long created, long expiryTime) {
+ this.name = name;
+ this.created = created;
+ this.expiryTime = expiryTime;
+ }
+
+ public static Checkpoint createFromSegmentNode(String name, NodeState
node) {
+ return new Checkpoint(name, node.getLong("created"),
node.getLong("timestamp"));
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public long getExpiryTime() {
+ return expiryTime;
+ }
+
+ @Override
+ public int compareTo(Checkpoint o) {
+ return Long.compare(this.created, o.created);
+ }
+ }
+
+ private CheckpointRetriever() {
+ }
+
+ public static List<Checkpoint> getCheckpoints(NodeStore nodeStore) {
+ List<Checkpoint> result;
+ if (nodeStore instanceof SegmentNodeStore) {
+ result =
getCheckpoints(CheckpointAccessor.getCheckpointsRoot((SegmentNodeStore)
nodeStore));
+ } else if (nodeStore instanceof
org.apache.jackrabbit.oak.plugins.segment.SegmentNodeStore) {
+ result =
getCheckpoints(org.apache.jackrabbit.oak.plugins.segment.CheckpointAccessor.getCheckpointsRoot((org.apache.jackrabbit.oak.plugins.segment.SegmentNodeStore)
nodeStore));
+ } else if (nodeStore instanceof DocumentNodeStore) {
+ result =
DocumentCheckpointRetriever.getCheckpoints((DocumentNodeStore) nodeStore);
+ } else if (nodeStore instanceof
SegmentTarFactory.NodeStoreWithFileStore) {
+ result =
getCheckpoints(CheckpointAccessor.getCheckpointsRoot(((SegmentTarFactory.NodeStoreWithFileStore)
nodeStore).getNodeStore()));
+ } else {
+ return null;
+ }
+ Collections.sort(result);
+ return result;
+ }
+
+ private static List<Checkpoint> getCheckpoints(NodeState checkpointRoot) {
+ return
Lists.newArrayList(Iterables.transform(checkpointRoot.getChildNodeEntries(),
new Function<ChildNodeEntry, Checkpoint>() {
+ @Nullable
+ @Override
+ public Checkpoint apply(@Nullable ChildNodeEntry input) {
+ return Checkpoint.createFromSegmentNode(input.getName(),
input.getNodeState());
+ }
+ }));
+ }
+}
\ No newline at end of file
Modified:
jackrabbit/oak/branches/1.6/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/MigrationFactory.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.6/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/MigrationFactory.java?rev=1815885&r1=1815884&r2=1815885&view=diff
==============================================================================
---
jackrabbit/oak/branches/1.6/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/MigrationFactory.java
(original)
+++
jackrabbit/oak/branches/1.6/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/MigrationFactory.java
Tue Nov 21 08:33:10 2017
@@ -18,6 +18,7 @@ package org.apache.jackrabbit.oak.upgrad
import java.io.IOException;
import java.util.Iterator;
+import java.util.List;
import java.util.ServiceLoader;
import javax.jcr.RepositoryException;
@@ -85,6 +86,12 @@ public class MigrationFactory {
if (options.getExcludePaths() != null) {
upgrade.setExcludes(options.getExcludePaths());
}
+ if (options.getFragmentPaths() != null) {
+ upgrade.setFragmentPaths(options.getFragmentPaths());
+ }
+ if (options.getExcludeFragments() != null) {
+ upgrade.setExcludeFragments(options.getExcludeFragments());
+ }
if (options.getMergePaths() != null) {
upgrade.setMerges(options.getMergePaths());
}
@@ -93,10 +100,7 @@ public class MigrationFactory {
upgrade.setSkipOnError(!options.isFailOnError());
upgrade.setEarlyShutdown(options.isEarlyShutdown());
upgrade.setSkipInitialization(options.isSkipInitialization());
- ServiceLoader<CommitHook> loader =
ServiceLoader.load(CommitHook.class);
- Iterator<CommitHook> iterator = loader.iterator();
- ImmutableList.Builder<CommitHook> builder = ImmutableList.<CommitHook>
builder().addAll(iterator);
- upgrade.setCustomCommitHooks(builder.build());
+ upgrade.setCustomCommitHooks(loacCommitHooks());
return upgrade;
}
@@ -110,6 +114,12 @@ public class MigrationFactory {
if (options.getExcludePaths() != null) {
sidegrade.setExcludes(options.getExcludePaths());
}
+ if (options.getExcludeFragments() != null) {
+ sidegrade.setExcludeFragments(options.getExcludeFragments());
+ }
+ if (options.getFragmentPaths() != null) {
+ sidegrade.setFragmentPaths(options.getFragmentPaths());
+ }
if (options.getMergePaths() != null) {
sidegrade.setMerges(options.getMergePaths());
}
@@ -118,7 +128,16 @@ public class MigrationFactory {
sidegrade.setVerify(options.isVerify());
sidegrade.setOnlyVerify(options.isOnlyVerify());
sidegrade.setSkipCheckpoints(options.isSkipCheckpoints());
+ sidegrade.setForceCheckpoints(options.isForceCheckpoints());
+ sidegrade.setCustomCommitHooks(loacCommitHooks());
return sidegrade;
}
+ private List<CommitHook> loacCommitHooks() {
+ ServiceLoader<CommitHook> loader =
ServiceLoader.load(CommitHook.class);
+ Iterator<CommitHook> iterator = loader.iterator();
+ ImmutableList.Builder<CommitHook> builder = ImmutableList.<CommitHook>
builder().addAll(iterator);
+ return builder.build();
+ }
+
}
Modified:
jackrabbit/oak/branches/1.6/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/node/JdbcFactory.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.6/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/node/JdbcFactory.java?rev=1815885&r1=1815884&r2=1815885&view=diff
==============================================================================
---
jackrabbit/oak/branches/1.6/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/node/JdbcFactory.java
(original)
+++
jackrabbit/oak/branches/1.6/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/node/JdbcFactory.java
Tue Nov 21 08:33:10 2017
@@ -58,6 +58,7 @@ public class JdbcFactory implements Node
@Override
public NodeStore create(BlobStore blobStore, Closer closer) throws
IOException {
+ System.setProperty(DocumentNodeStore.SYS_PROP_DISABLE_JOURNAL, "true");
DocumentMK.Builder builder = MongoFactory.getBuilder(cacheSize);
if (blobStore != null) {
builder.setBlobStore(blobStore);
Modified:
jackrabbit/oak/branches/1.6/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/node/MongoFactory.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.6/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/node/MongoFactory.java?rev=1815885&r1=1815884&r2=1815885&view=diff
==============================================================================
---
jackrabbit/oak/branches/1.6/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/node/MongoFactory.java
(original)
+++
jackrabbit/oak/branches/1.6/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/node/MongoFactory.java
Tue Nov 21 08:33:10 2017
@@ -49,6 +49,7 @@ public class MongoFactory implements Nod
@Override
public NodeStore create(BlobStore blobStore, Closer closer) throws
IOException {
+ System.setProperty(DocumentNodeStore.SYS_PROP_DISABLE_JOURNAL, "true");
DocumentMK.Builder builder = getBuilder(cacheSize);
builder.setMongoDB(getDB(closer));
if (blobStore != null) {
Modified:
jackrabbit/oak/branches/1.6/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/node/SegmentFactory.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.6/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/node/SegmentFactory.java?rev=1815885&r1=1815884&r2=1815885&view=diff
==============================================================================
---
jackrabbit/oak/branches/1.6/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/node/SegmentFactory.java
(original)
+++
jackrabbit/oak/branches/1.6/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/node/SegmentFactory.java
Tue Nov 21 08:33:10 2017
@@ -21,23 +21,17 @@ import java.io.File;
import java.io.IOException;
import org.apache.jackrabbit.oak.plugins.blob.ReferenceCollector;
-import org.apache.jackrabbit.oak.plugins.segment.SegmentNodeBuilder;
-import org.apache.jackrabbit.oak.plugins.segment.SegmentNodeState;
import org.apache.jackrabbit.oak.plugins.segment.SegmentNodeStore;
import org.apache.jackrabbit.oak.plugins.segment.file.FileStore;
import org.apache.jackrabbit.oak.plugins.segment.file.FileStore.Builder;
import
org.apache.jackrabbit.oak.plugins.segment.file.InvalidFileStoreVersionException;
import org.apache.jackrabbit.oak.spi.blob.BlobStore;
-import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
-import org.apache.jackrabbit.oak.spi.state.NodeState;
import org.apache.jackrabbit.oak.spi.state.NodeStore;
import com.google.common.io.Closer;
import javax.annotation.Nullable;
-import static com.google.common.base.Preconditions.checkArgument;
-
public class SegmentFactory implements NodeStoreFactory {
private final File dir;
@@ -87,25 +81,7 @@ public class SegmentFactory implements N
}
closer.register(asCloseable(fs));
- return new TarNodeStore(SegmentNodeStore.builder(fs).build(), new
TarNodeStore.SuperRootProvider() {
- @Override
- public void setSuperRoot(NodeBuilder builder) {
- checkArgument(builder instanceof SegmentNodeBuilder);
- SegmentNodeBuilder segmentBuilder = (SegmentNodeBuilder)
builder;
- SegmentNodeState lastRoot = (SegmentNodeState) getSuperRoot();
-
- if (!lastRoot.getRecordId().equals(((SegmentNodeState)
segmentBuilder.getBaseState()).getRecordId())) {
- throw new IllegalArgumentException("The new head is out of
date");
- }
-
- fs.setHead(lastRoot, ((SegmentNodeBuilder)
builder).getNodeState());
- }
-
- @Override
- public NodeState getSuperRoot() {
- return fs.getHead();
- }
- });
+ return SegmentNodeStore.builder(fs).build();
}
@Override
Modified:
jackrabbit/oak/branches/1.6/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/node/SegmentTarFactory.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.6/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/node/SegmentTarFactory.java?rev=1815885&r1=1815884&r2=1815885&view=diff
==============================================================================
---
jackrabbit/oak/branches/1.6/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/node/SegmentTarFactory.java
(original)
+++
jackrabbit/oak/branches/1.6/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/node/SegmentTarFactory.java
Tue Nov 21 08:33:10 2017
@@ -16,7 +16,6 @@
*/
package org.apache.jackrabbit.oak.upgrade.cli.node;
-import static com.google.common.base.Preconditions.checkArgument;
import static
org.apache.jackrabbit.oak.segment.file.FileStoreBuilder.fileStoreBuilder;
import java.io.Closeable;
@@ -27,17 +26,15 @@ import com.google.common.io.Closer;
import org.apache.jackrabbit.oak.segment.RecordType;
import org.apache.jackrabbit.oak.segment.Segment;
import org.apache.jackrabbit.oak.segment.SegmentId;
-import org.apache.jackrabbit.oak.segment.SegmentNodeBuilder;
-import org.apache.jackrabbit.oak.segment.SegmentNodeState;
+import org.apache.jackrabbit.oak.segment.SegmentNodeStore;
import org.apache.jackrabbit.oak.segment.SegmentNodeStoreBuilders;
import org.apache.jackrabbit.oak.segment.file.FileStore;
import org.apache.jackrabbit.oak.segment.file.FileStoreBuilder;
import org.apache.jackrabbit.oak.segment.file.InvalidFileStoreVersionException;
import org.apache.jackrabbit.oak.segment.file.ReadOnlyFileStore;
import org.apache.jackrabbit.oak.spi.blob.BlobStore;
-import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
-import org.apache.jackrabbit.oak.spi.state.NodeState;
import org.apache.jackrabbit.oak.spi.state.NodeStore;
+import org.apache.jackrabbit.oak.spi.state.ProxyNodeStore;
public class SegmentTarFactory implements NodeStoreFactory {
@@ -81,12 +78,12 @@ public class SegmentTarFactory implement
final ReadOnlyFileStore fs;
fs = builder.buildReadOnly();
closer.register(asCloseable(fs));
- return new
TarNodeStore(SegmentNodeStoreBuilders.builder(fs).build(), new
SegmentTarSuperRootProvider(fs));
+ return SegmentNodeStoreBuilders.builder(fs).build();
} else {
final FileStore fs;
fs = builder.build();
closer.register(asCloseable(fs));
- return new
TarNodeStore(SegmentNodeStoreBuilders.builder(fs).build(), new
SegmentTarSuperRootProvider(fs));
+ return new
NodeStoreWithFileStore(SegmentNodeStoreBuilders.builder(fs).build(), fs);
}
} catch (InvalidFileStoreVersionException e) {
throw new IllegalStateException(e);
@@ -158,40 +155,24 @@ public class SegmentTarFactory implement
private static class ExternalBlobFound extends RuntimeException {
}
- private static class SegmentTarSuperRootProvider implements
TarNodeStore.SuperRootProvider {
+ public static class NodeStoreWithFileStore extends ProxyNodeStore {
- private final ReadOnlyFileStore readOnlyFileStore;
+ private final SegmentNodeStore segmentNodeStore;
private final FileStore fileStore;
- public SegmentTarSuperRootProvider(ReadOnlyFileStore
readOnlyFileStore) {
- this.readOnlyFileStore = readOnlyFileStore;
- this.fileStore = null;
- }
-
- public SegmentTarSuperRootProvider(FileStore fileStore) {
- this.readOnlyFileStore = null;
+ public NodeStoreWithFileStore(SegmentNodeStore segmentNodeStore,
FileStore fileStore) {
+ this.segmentNodeStore = segmentNodeStore;
this.fileStore = fileStore;
}
- @Override
- public void setSuperRoot(NodeBuilder builder) {
- if (fileStore == null) {
- throw new IllegalStateException("setSuperRoot is not supported
for read-only segment-tar");
- }
- checkArgument(builder instanceof SegmentNodeBuilder);
- SegmentNodeBuilder segmentBuilder = (SegmentNodeBuilder) builder;
- SegmentNodeState lastRoot = (SegmentNodeState) getSuperRoot();
-
- if (!lastRoot.getRecordId().equals(((SegmentNodeState)
segmentBuilder.getBaseState()).getRecordId())) {
- throw new IllegalArgumentException("The new head is out of
date");
- }
- fileStore.getRevisions().setHead(lastRoot.getRecordId(),
segmentBuilder.getNodeState().getRecordId());
+ public FileStore getFileStore() {
+ return fileStore;
}
@Override
- public NodeState getSuperRoot() {
- return fileStore == null ? readOnlyFileStore.getHead() :
fileStore.getHead();
+ public SegmentNodeStore getNodeStore() {
+ return segmentNodeStore;
}
}
}
Modified:
jackrabbit/oak/branches/1.6/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/parser/MigrationCliArguments.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.6/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/parser/MigrationCliArguments.java?rev=1815885&r1=1815884&r2=1815885&view=diff
==============================================================================
---
jackrabbit/oak/branches/1.6/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/parser/MigrationCliArguments.java
(original)
+++
jackrabbit/oak/branches/1.6/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/parser/MigrationCliArguments.java
Tue Nov 21 08:33:10 2017
@@ -52,6 +52,10 @@ public final class MigrationCliArguments
return (Integer) options.valueOf(optionName);
}
+ public Boolean getBooleanOption(String optionName) {
+ return (Boolean) options.valueOf(optionName);
+ }
+
public String[] getOptionList(String optionName) {
String option = getOption(optionName);
if (option == null) {
Modified:
jackrabbit/oak/branches/1.6/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/parser/MigrationOptions.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.6/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/parser/MigrationOptions.java?rev=1815885&r1=1815884&r2=1815885&view=diff
==============================================================================
---
jackrabbit/oak/branches/1.6/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/parser/MigrationOptions.java
(original)
+++
jackrabbit/oak/branches/1.6/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/parser/MigrationOptions.java
Tue Nov 21 08:33:10 2017
@@ -46,6 +46,10 @@ public class MigrationOptions {
private final String[] excludePaths;
+ private final String[] fragmentPaths;
+
+ private final String[] excludeFragments;
+
private final String[] mergePaths;
private final boolean includeIndex;
@@ -66,6 +70,8 @@ public class MigrationOptions {
private final boolean skipCheckpoints;
+ private final boolean forceCheckpoints;
+
private final String srcUser;
private final String srcPassword;
@@ -115,6 +121,8 @@ public class MigrationOptions {
}
this.includePaths =
checkPaths(args.getOptionList(OptionParserFactory.INCLUDE_PATHS));
this.excludePaths =
checkPaths(args.getOptionList(OptionParserFactory.EXCLUDE_PATHS));
+ this.fragmentPaths =
checkPaths(args.getOptionList(OptionParserFactory.FRAGMENT_PATHS));
+ this.excludeFragments =
args.getOptionList(OptionParserFactory.EXCLUDE_FRAGMENTS);
this.mergePaths =
checkPaths(args.getOptionList(OptionParserFactory.MERGE_PATHS));
this.includeIndex = args.hasOption(OptionParserFactory.INCLUDE_INDEX);
this.failOnError = args.hasOption(OptionParserFactory.FAIL_ON_ERROR);
@@ -125,6 +133,7 @@ public class MigrationOptions {
this.verify = args.hasOption(OptionParserFactory.VERIFY);
this.onlyVerify = args.hasOption(OptionParserFactory.ONLY_VERIFY);
this.skipCheckpoints =
args.hasOption(OptionParserFactory.SKIP_CHECKPOINTS);
+ this.forceCheckpoints =
args.hasOption(OptionParserFactory.FORCE_CHECKPOINTS);
this.srcUser = args.getOption(OptionParserFactory.SRC_USER);
this.srcPassword = args.getOption(OptionParserFactory.SRC_USER);
@@ -142,8 +151,7 @@ public class MigrationOptions {
this.dstS3Config = args.getOption(OptionParserFactory.DST_S3_CONFIG);
if (args.hasOption(OptionParserFactory.SRC_EXTERNAL_BLOBS)) {
- this.srcExternalBlobs = Boolean
- .valueOf(OptionParserFactory.SRC_EXTERNAL_BLOBS);
+ this.srcExternalBlobs =
args.getBooleanOption(OptionParserFactory.SRC_EXTERNAL_BLOBS);
} else {
this.srcExternalBlobs = null;
}
@@ -177,6 +185,14 @@ public class MigrationOptions {
return excludePaths;
}
+ public String[] getFragmentPaths() {
+ return fragmentPaths;
+ }
+
+ public String[] getExcludeFragments() {
+ return excludeFragments;
+ }
+
public String[] getMergePaths() {
return mergePaths;
}
@@ -217,6 +233,10 @@ public class MigrationOptions {
return skipCheckpoints;
}
+ public boolean isForceCheckpoints() {
+ return forceCheckpoints;
+ }
+
public String getSrcUser() {
return srcUser;
}
@@ -322,6 +342,14 @@ public class MigrationOptions {
log.info("paths to exclude: {}", (Object) excludePaths);
}
+ if (fragmentPaths != null) {
+ log.info("paths supporting fragments: {}", (Object) fragmentPaths);
+ }
+
+ if (excludeFragments != null) {
+ log.info("fragments to exclude: {}", (Object) excludeFragments);
+ }
+
if (failOnError) {
log.info("Unreadable nodes will cause failure of the entire
transaction");
}
@@ -354,6 +382,10 @@ public class MigrationOptions {
log.info("Checkpoints won't be migrated");
}
+ if (forceCheckpoints) {
+ log.info("Checkpoints will be migrated even with the custom paths
specified");
+ }
+
log.info("Cache size: {} MB", cacheSizeInMB);
}
Modified:
jackrabbit/oak/branches/1.6/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/parser/OptionParserFactory.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.6/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/parser/OptionParserFactory.java?rev=1815885&r1=1815884&r2=1815885&view=diff
==============================================================================
---
jackrabbit/oak/branches/1.6/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/parser/OptionParserFactory.java
(original)
+++
jackrabbit/oak/branches/1.6/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/parser/OptionParserFactory.java
Tue Nov 21 08:33:10 2017
@@ -70,6 +70,10 @@ public class OptionParserFactory {
public static final String EXCLUDE_PATHS = "exclude-paths";
+ public static final String FRAGMENT_PATHS = "fragment-paths";
+
+ public static final String EXCLUDE_FRAGMENTS = "exclude-fragments";
+
public static final String MERGE_PATHS = "merge-paths";
public static final String SKIP_INIT = "skip-init";
@@ -84,6 +88,8 @@ public class OptionParserFactory {
public static final String SKIP_CHECKPOINTS = "skip-checkpoints";
+ public static final String FORCE_CHECKPOINTS = "force-checkpoints";
+
public static OptionParser create() {
OptionParser op = new OptionParser();
addUsageOptions(op);
@@ -116,7 +122,8 @@ public class OptionParserFactory {
op.accepts(DST_S3_CONFIG, "Configuration file for the target
S3DataStore").withRequiredArg()
.ofType(String.class);
op.accepts(IGNORE_MISSING_BINARIES, "Don't break the migration if some
binaries are missing");
- op.accepts(SRC_EXTERNAL_BLOBS, "Flag specifying if the source Store
has external references or not");
+ op.accepts(SRC_EXTERNAL_BLOBS, "Flag specifying if the source Store
has external references or not")
+ .withRequiredArg().ofType(Boolean.class);
}
private static void addRdbOptions(OptionParser op) {
@@ -131,6 +138,10 @@ public class OptionParserFactory {
.ofType(String.class);
op.accepts(EXCLUDE_PATHS, "Comma-separated list of paths to exclude
during copy.").withRequiredArg()
.ofType(String.class);
+ op.accepts(FRAGMENT_PATHS, "Comma-separated list of paths supporting
fragments.").withRequiredArg()
+ .ofType(String.class);
+ op.accepts(EXCLUDE_FRAGMENTS, "Comma-separated list of fragments to
exclude during copy.").withRequiredArg()
+ .ofType(String.class);
op.accepts(MERGE_PATHS, "Comma-separated list of paths to merge during
copy.").withRequiredArg()
.ofType(String.class);
op.accepts(INCLUDE_INDEX, "Copy index data for paths specified in the
" + INCLUDE_PATHS + " option");
@@ -156,5 +167,6 @@ public class OptionParserFactory {
op.accepts(VERIFY, "After the sidegrade check whether the source
repository is exactly the same as destination");
op.accepts(ONLY_VERIFY, "Performs only --" + VERIFY + ", without
copying content");
op.accepts(SKIP_CHECKPOINTS, "Don't copy checkpoints on the full
segment->segment migration");
+ op.accepts(FORCE_CHECKPOINTS, "Copy checkpoints even if the
--include,exclude,merge-paths option is specified");
}
}
