Author: amitj
Date: Wed Sep 6 09:05:57 2017
New Revision: 1807453
URL: http://svn.apache.org/viewvc?rev=1807453&view=rev
Log:
OAK-6611: [upgrade][oak-blob-cloud] Many S3DataStore errors during migration
with oak-upgrade
- Initialize S3DataStore with a default metrics stats provider to have
meaningful stats
- Added a test
Added:
jackrabbit/oak/trunk/oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/cli/blob/S3ToInlinedTest.java
(with props)
Modified:
jackrabbit/oak/trunk/oak-upgrade/pom.xml
jackrabbit/oak/trunk/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/blob/S3DataStoreFactory.java
Modified: jackrabbit/oak/trunk/oak-upgrade/pom.xml
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-upgrade/pom.xml?rev=1807453&r1=1807452&r2=1807453&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-upgrade/pom.xml (original)
+++ jackrabbit/oak/trunk/oak-upgrade/pom.xml Wed Sep 6 09:05:57 2017
@@ -171,6 +171,13 @@
<type>test-jar</type>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.apache.jackrabbit</groupId>
+ <artifactId>oak-commons</artifactId>
+ <version>${project.version}</version>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
<dependency>
<groupId>org.hamcrest</groupId>
Modified:
jackrabbit/oak/trunk/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/blob/S3DataStoreFactory.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/blob/S3DataStoreFactory.java?rev=1807453&r1=1807452&r2=1807453&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/blob/S3DataStoreFactory.java
(original)
+++
jackrabbit/oak/trunk/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/blob/S3DataStoreFactory.java
Wed Sep 6 09:05:57 2017
@@ -22,6 +22,7 @@ import java.io.FileReader;
import java.io.IOException;
import java.util.HashSet;
import java.util.Properties;
+import java.util.concurrent.Executors;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@@ -35,6 +36,8 @@ import org.apache.jackrabbit.oak.spi.blo
import com.google.common.io.Closer;
import com.google.common.io.Files;
+import org.apache.jackrabbit.oak.stats.DefaultStatisticsProvider;
+import org.apache.jackrabbit.oak.stats.StatisticsProvider;
public class S3DataStoreFactory implements BlobStoreFactory {
@@ -72,6 +75,13 @@ public class S3DataStoreFactory implemen
S3DataStore delegate = new S3DataStore();
delegate.setProperties(props);
delegate.setPath(directory);
+
+ // Initialize a default stats provider
+ StatisticsProvider statsProvider = new
DefaultStatisticsProvider(Executors.newSingleThreadScheduledExecutor());
+ delegate.setStatisticsProvider(statsProvider);
+ // Reduce staging purge interval to 60 seconds
+ delegate.setStagingPurgeInterval(60);
+
try {
delegate.init(tempHomeDir.getPath());
} catch (RepositoryException e) {
Added:
jackrabbit/oak/trunk/oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/cli/blob/S3ToInlinedTest.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/cli/blob/S3ToInlinedTest.java?rev=1807453&view=auto
==============================================================================
---
jackrabbit/oak/trunk/oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/cli/blob/S3ToInlinedTest.java
(added)
+++
jackrabbit/oak/trunk/oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/cli/blob/S3ToInlinedTest.java
Wed Sep 6 09:05:57 2017
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.jackrabbit.oak.upgrade.cli.blob;
+
+import java.io.IOException;
+
+import ch.qos.logback.classic.Level;
+import org.apache.jackrabbit.oak.commons.junit.LogCustomizer;
+import org.apache.jackrabbit.oak.plugins.blob.UploadStagingCache;
+import org.apache.jackrabbit.oak.upgrade.cli.AbstractOak2OakTest;
+import org.apache.jackrabbit.oak.upgrade.cli.container.BlobStoreContainer;
+import org.apache.jackrabbit.oak.upgrade.cli.container.NodeStoreContainer;
+import org.apache.jackrabbit.oak.upgrade.cli.container.S3DataStoreContainer;
+import
org.apache.jackrabbit.oak.upgrade.cli.container.SegmentTarNodeStoreContainer;
+import org.junit.Assume;
+import org.junit.Before;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+
+public class S3ToInlinedTest extends AbstractOak2OakTest {
+ private static final String S3_PROPERTIES =
System.getProperty("s3.properties");
+
+ private final NodeStoreContainer source;
+
+ private final NodeStoreContainer destination;
+
+ private final BlobStoreContainer sourceBlob;
+
+ private LogCustomizer customLogs;
+
+ public S3ToInlinedTest() throws IOException {
+ Assume.assumeTrue(S3_PROPERTIES != null && !S3_PROPERTIES.isEmpty());
+ sourceBlob = new S3DataStoreContainer(S3_PROPERTIES);
+ source = new SegmentTarNodeStoreContainer(sourceBlob);
+ destination = new SegmentTarNodeStoreContainer();
+ }
+
+ @Override
+ protected NodeStoreContainer getSourceContainer() {
+ return source;
+ }
+
+ @Override
+ protected NodeStoreContainer getDestinationContainer() {
+ return destination;
+ }
+
+ @Override
+ protected String[] getArgs() {
+ return new String[] { "--copy-binaries", "--src-s3datastore",
sourceBlob.getDescription(), "--src-s3config",
+ S3_PROPERTIES, source.getDescription(),
destination.getDescription() };
+ }
+
+ @Override
+ protected boolean supportsCheckpointMigration() {
+ return true;
+ }
+
+ @Before
+ public void prepare() throws Exception {
+ // Capture logs
+ customLogs = LogCustomizer
+ .forLogger(UploadStagingCache.class.getName())
+ .enable(Level.INFO)
+ .filter(Level.INFO)
+ .contains("Uploads in progress on close [0]")
+ .create();
+ customLogs.starting();
+ super.prepare();
+ }
+
+ /**
+ * Tests whether all the s3 uploads finished
+ */
+ @Test
+ public void testAsyncUploadFinished() {
+ assertEquals(1, customLogs.getLogs().size());
+ customLogs.finished();
+ }
+
+}
Propchange:
jackrabbit/oak/trunk/oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/cli/blob/S3ToInlinedTest.java
------------------------------------------------------------------------------
svn:eol-style = native
