Author: amitj
Date: Thu May 18 10:00:08 2017
New Revision: 1795497
URL: http://svn.apache.org/viewvc?rev=1795497&view=rev
Log:
OAK-6229: NPE when running datastorecheck command with S3
- Merge r1795475 from trunk
- Added explicit dependency to httpcore-4.4.4 and httpclient-4.5.2 as stale
versions not supported by the latest aws-java-sdk were being pulled in by solr
- Added explicit dependency to jackson core, annotations and databind as stale
versions not supported by the latest aws-java-sdk, were being pulled in by
oak-http
Modified:
jackrabbit/oak/branches/1.6/ (props changed)
jackrabbit/oak/branches/1.6/oak-run/pom.xml
jackrabbit/oak/branches/1.6/oak-run/src/main/java/org/apache/jackrabbit/oak/run/Utils.java
jackrabbit/oak/branches/1.6/oak-run/src/test/java/org/apache/jackrabbit/oak/run/DataStoreCheckTest.java
Propchange: jackrabbit/oak/branches/1.6/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu May 18 10:00:08 2017
@@ -1,3 +1,3 @@
/jackrabbit/oak/branches/1.0:1665962
-/jackrabbit/oak/trunk:1781068,1781075,1781248,1781386,1781846,1781907,1782000,1782029,1782196,1782447,1782476,1782770,1782945,1782966,1782973,1782990,1783061,1783066,1783089,1783104-1783105,1783110,1783619,1783720,1783731,1783733,1783738,1783742,1783773,1783855,1783891,1784023,1784130,1784162,1784251,1784401,1784551,1784574,1784689,1785095,1785108,1785283,1785838,1785919,1785946,1787074,1787217,1787425,1788378,1788387-1788389,1788850,1789056,1790382,1792463,1792742,1792746,1793088,1793618,1793627,1793644,1795314,1795330
+/jackrabbit/oak/trunk:1781068,1781075,1781248,1781386,1781846,1781907,1782000,1782029,1782196,1782447,1782476,1782770,1782945,1782966,1782973,1782990,1783061,1783066,1783089,1783104-1783105,1783110,1783619,1783720,1783731,1783733,1783738,1783742,1783773,1783855,1783891,1784023,1784130,1784162,1784251,1784401,1784551,1784574,1784689,1785095,1785108,1785283,1785838,1785919,1785946,1787074,1787217,1787425,1788378,1788387-1788389,1788850,1789056,1790382,1792463,1792742,1792746,1793088,1793618,1793627,1793644,1795314,1795330,1795475
/jackrabbit/trunk:1345480
Modified: jackrabbit/oak/branches/1.6/oak-run/pom.xml
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.6/oak-run/pom.xml?rev=1795497&r1=1795496&r2=1795497&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.6/oak-run/pom.xml (original)
+++ jackrabbit/oak/branches/1.6/oak-run/pom.xml Thu May 18 10:00:08 2017
@@ -411,14 +411,62 @@
<groupId>com.google.code.findbugs</groupId>
<artifactId>jsr305</artifactId>
</dependency>
-
+
+ <!-- httpcore & httpclient -->
+ <dependency>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpcore</artifactId>
+ <version>4.4.4</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpclient</artifactId>
+ <version>4.5.2</version>
+ </dependency>
+
+ <!-- jackson libs -->
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-annotations</artifactId>
+ <version>2.6.6</version>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-core</artifactId>
+ <version>2.6.6</version>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ <version>2.6.6</version>
+ </dependency>
<!-- Test dependencies -->
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<scope>test</scope>
</dependency>
-
+ <dependency>
+ <groupId>org.apache.jackrabbit</groupId>
+ <artifactId>oak-core</artifactId>
+ <version>${project.version}</version>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.jackrabbit</groupId>
+ <artifactId>oak-blob-cloud</artifactId>
+ <version>${project.version}</version>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.jackrabbit</groupId>
+ <artifactId>oak-blob-cloud-azure</artifactId>
+ <version>${project.version}</version>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
</dependencies>
<profiles>
Modified:
jackrabbit/oak/branches/1.6/oak-run/src/main/java/org/apache/jackrabbit/oak/run/Utils.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.6/oak-run/src/main/java/org/apache/jackrabbit/oak/run/Utils.java?rev=1795497&r1=1795496&r2=1795497&view=diff
==============================================================================
---
jackrabbit/oak/branches/1.6/oak-run/src/main/java/org/apache/jackrabbit/oak/run/Utils.java
(original)
+++
jackrabbit/oak/branches/1.6/oak-run/src/main/java/org/apache/jackrabbit/oak/run/Utils.java
Thu May 18 10:00:08 2017
@@ -151,7 +151,9 @@ class Utils {
String cfgPath = s3dsConfig.value(options);
Properties props = loadAndTransformProps(cfgPath);
s3ds.setProperties(props);
- s3ds.init(null);
+ File homeDir = Files.createTempDir();
+ closer.register(asCloseable(homeDir));
+ s3ds.init(homeDir.getAbsolutePath());
delegate = s3ds;
} else if (options.has(azureBlobDSConfig)) {
AzureDataStore azureds = new AzureDataStore();
Modified:
jackrabbit/oak/branches/1.6/oak-run/src/test/java/org/apache/jackrabbit/oak/run/DataStoreCheckTest.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.6/oak-run/src/test/java/org/apache/jackrabbit/oak/run/DataStoreCheckTest.java?rev=1795497&r1=1795496&r2=1795497&view=diff
==============================================================================
---
jackrabbit/oak/branches/1.6/oak-run/src/test/java/org/apache/jackrabbit/oak/run/DataStoreCheckTest.java
(original)
+++
jackrabbit/oak/branches/1.6/oak-run/src/test/java/org/apache/jackrabbit/oak/run/DataStoreCheckTest.java
Thu May 18 10:00:08 2017
@@ -21,7 +21,6 @@ package org.apache.jackrabbit.oak.run;
import static com.google.common.base.Charsets.UTF_8;
import static org.junit.Assert.assertEquals;
-import java.io.BufferedWriter;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.File;
@@ -34,6 +33,7 @@ import java.io.PrintStream;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
+import java.util.Properties;
import java.util.Random;
import java.util.Set;
@@ -41,9 +41,12 @@ import com.google.common.collect.Immutab
import com.google.common.collect.Iterables;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
-import com.google.common.io.Files;
+import org.apache.commons.io.FileUtils;
import org.apache.commons.io.filefilter.FileFilterUtils;
-import org.apache.commons.lang3.StringEscapeUtils;
+import org.apache.felix.cm.file.ConfigurationHandler;
+import org.apache.jackrabbit.core.data.DataStore;
+import
org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureDataStoreUtils;
+import org.apache.jackrabbit.oak.blob.cloud.s3.S3DataStoreUtils;
import org.apache.jackrabbit.oak.commons.FileIOUtils;
import org.apache.jackrabbit.oak.plugins.blob.datastore.DataStoreBlobStore;
import org.apache.jackrabbit.oak.plugins.blob.datastore.OakFileDataStore;
@@ -81,18 +84,48 @@ public class DataStoreCheckTest {
private String dsPath;
+ private DataStoreBlobStore setupDataStore;
+
+ private String dsOption;
+
@Before
public void setup() throws Exception {
- OakFileDataStore delegate = new OakFileDataStore();
- dsPath = temporaryFolder.newFolder().getAbsolutePath();
- delegate.setPath(dsPath);
- delegate.init(null);
- DataStoreBlobStore blobStore = new DataStoreBlobStore(delegate);
+ if (S3DataStoreUtils.isS3Configured()) {
+ Properties props = S3DataStoreUtils.getS3Config();
+ props.setProperty("cacheSize", "0");
+ DataStore ds =
S3DataStoreUtils.getS3DataStore(S3DataStoreUtils.getFixtures().get(0),
+ props,
+ temporaryFolder.newFolder().getAbsolutePath());
+ setupDataStore = new DataStoreBlobStore(ds);
+ cfgFilePath = createTempConfig(temporaryFolder.newFile(), props);
+ dsOption = "s3ds";
+ } else if (AzureDataStoreUtils.isAzureConfigured()) {
+ Properties props = AzureDataStoreUtils.getAzureConfig();
+ props.setProperty("cacheSize", "0");
+ DataStore ds = AzureDataStoreUtils.getAzureDataStore(props,
+ temporaryFolder.newFolder().getAbsolutePath());
+ setupDataStore = new DataStoreBlobStore(ds);
+ cfgFilePath = createTempConfig(temporaryFolder.newFile(), props);
+ dsOption = "azureblobds";
+ }
+ else {
+ OakFileDataStore delegate = new OakFileDataStore();
+ dsPath = temporaryFolder.newFolder().getAbsolutePath();
+ delegate.setPath(dsPath);
+ delegate.init(null);
+ setupDataStore = new DataStoreBlobStore(delegate);
+
+ File cfgFile = temporaryFolder.newFile();
+ Properties props = new Properties();
+ props.put("path", dsPath);
+ cfgFilePath = createTempConfig(cfgFile, props);
+ dsOption = "fds";
+ }
File storeFile = temporaryFolder.newFolder();
storePath = storeFile.getAbsolutePath();
FileStore fileStore = FileStoreBuilder.fileStoreBuilder(storeFile)
- .withBlobStore(blobStore)
+ .withBlobStore(setupDataStore)
.withMaxFileSize(256)
.withSegmentCacheSize(64)
.build();
@@ -104,7 +137,7 @@ public class DataStoreCheckTest {
blobsAdded = Sets.newHashSet();
for (int i = 0; i < numBlobs; i++) {
SegmentBlob b = (SegmentBlob) store.createBlob(randomStream(i,
18342));
- Iterator<String> idIter = blobStore.resolveChunks(b.getBlobId());
+ Iterator<String> idIter =
setupDataStore.resolveChunks(b.getBlobId());
while (idIter.hasNext()) {
String chunk = idIter.next();
blobsAdded.add(chunk);
@@ -115,14 +148,7 @@ public class DataStoreCheckTest {
store.merge(a, EmptyHook.INSTANCE, CommitInfo.EMPTY);
log.info("Created blobs : {}", blobsAdded);
- File cfgFile = temporaryFolder.newFile();
- BufferedWriter writer = Files.newWriter(cfgFile, UTF_8);
- FileIOUtils.writeAsLine(writer, "path=\"" +
StringEscapeUtils.escapeJava(dsPath) + "\"",false);
- writer.close();
- cfgFilePath = cfgFile.getAbsolutePath();
-
fileStore.close();
- blobStore.close();
}
@After
@@ -133,6 +159,7 @@ public class DataStoreCheckTest {
@Test
public void testCorrect() throws Exception {
File dump = temporaryFolder.newFolder();
+ setupDataStore.close();
testAllParams(dump);
}
@@ -140,17 +167,12 @@ public class DataStoreCheckTest {
public void testConsistency() throws Exception {
File dump = temporaryFolder.newFolder();
- // Delete a random blob from datastore
- OakFileDataStore delegate = new OakFileDataStore();
- delegate.setPath(dsPath);
- delegate.init(null);
- DataStoreBlobStore blobStore = new DataStoreBlobStore(delegate);
-
Random rand = new Random();
String deletedBlobId = Iterables.get(blobsAdded,
rand.nextInt(blobsAdded.size()));
blobsAdded.remove(deletedBlobId);
- long count =
blobStore.countDeleteChunks(ImmutableList.of(deletedBlobId), 0);
+ long count =
setupDataStore.countDeleteChunks(ImmutableList.of(deletedBlobId), 0);
assertEquals(1, count);
+ setupDataStore.close();
testAllParams(dump);
@@ -159,10 +181,10 @@ public class DataStoreCheckTest {
assertFileEquals(dump, "[consistency]",
Sets.newHashSet(deletedBlobId));
}
- public void testAllParams(File dump) throws Exception {
+ private void testAllParams(File dump) throws Exception {
DataStoreCheckCommand checkCommand = new DataStoreCheckCommand();
List<String> argsList = Lists
- .newArrayList("--id", "--ref", "--consistency", "--fds",
cfgFilePath, "--store", storePath,
+ .newArrayList("--id", "--ref", "--consistency", "--" + dsOption,
cfgFilePath, "--store", storePath,
"--dump", dump.getAbsolutePath());
checkCommand.execute(argsList.toArray(new String[0]));
@@ -170,9 +192,10 @@ public class DataStoreCheckTest {
@Test
public void testMissingOpParams() throws Exception {
+ setupDataStore.close();
File dump = temporaryFolder.newFolder();
List<String> argsList = Lists
- .newArrayList("--fds", cfgFilePath, "--store", storePath,
+ .newArrayList("--" + dsOption, cfgFilePath, "--store", storePath,
"--dump", dump.getAbsolutePath());
log.info("Running testMissinOpParams: {}", argsList);
testIncorrectParams(argsList, Lists.newArrayList("Missing required
option(s)", "'id'", "'ref'", "'consistency'"));
@@ -180,6 +203,7 @@ public class DataStoreCheckTest {
@Test
public void testTarNoDS() throws Exception {
+ setupDataStore.close();
File dump = temporaryFolder.newFolder();
List<String> argsList = Lists
.newArrayList("--id", "--ref", "--consistency", "--store",
storePath,
@@ -190,14 +214,15 @@ public class DataStoreCheckTest {
@Test
public void testOpNoStore() throws Exception {
+ setupDataStore.close();
File dump = temporaryFolder.newFolder();
List<String> argsList = Lists
- .newArrayList("--consistency", "--fds", cfgFilePath,
+ .newArrayList("--consistency", "--" + dsOption, cfgFilePath,
"--dump", dump.getAbsolutePath());
testIncorrectParams(argsList, Lists.newArrayList("Missing required
option(s) ['store']"));
argsList = Lists
- .newArrayList("--ref", "--fds", cfgFilePath,
+ .newArrayList("--ref", "--" + dsOption, cfgFilePath,
"--dump", dump.getAbsolutePath());
testIncorrectParams(argsList, Lists.newArrayList("Missing required
option(s) ['store']"));
}
@@ -236,4 +261,10 @@ public class DataStoreCheckTest {
r.nextBytes(data);
return new ByteArrayInputStream(data);
}
+
+ private static String createTempConfig(File cfgFile, Properties props)
throws IOException {
+ FileOutputStream fos = FileUtils.openOutputStream(cfgFile);
+ ConfigurationHandler.write(fos, props);
+ return cfgFile.getAbsolutePath();
+ }
}
