Author: amitj
Date: Thu May 18 03:36:19 2017
New Revision: 1795475
URL: http://svn.apache.org/viewvc?rev=1795475&view=rev
Log:
OAK-6229: NPE when running datastorecheck command with S3
- Initializing the S3DataStore with a temp directory
- Changed setup to also enable testing for S3 & Azure and test dependencies to
s3, azure modules
Modified:
jackrabbit/oak/trunk/oak-run/pom.xml
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/DataStoreCheckCommand.java
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/Utils.java
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/cli/BlobStoreFixtureProvider.java
jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/run/DataStoreCheckTest.java
Modified: jackrabbit/oak/trunk/oak-run/pom.xml
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/pom.xml?rev=1795475&r1=1795474&r2=1795475&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-run/pom.xml (original)
+++ jackrabbit/oak/trunk/oak-run/pom.xml Thu May 18 03:36:19 2017
@@ -319,7 +319,27 @@
<artifactId>junit</artifactId>
<scope>test</scope>
</dependency>
-
+ <dependency>
+ <groupId>org.apache.jackrabbit</groupId>
+ <artifactId>oak-blob-plugins</artifactId>
+ <version>${project.version}</version>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.jackrabbit</groupId>
+ <artifactId>oak-blob-cloud</artifactId>
+ <version>${project.version}</version>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.jackrabbit</groupId>
+ <artifactId>oak-blob-cloud-azure</artifactId>
+ <version>${project.version}</version>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
</dependencies>
<profiles>
Modified:
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/DataStoreCheckCommand.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/DataStoreCheckCommand.java?rev=1795475&r1=1795474&r2=1795475&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/DataStoreCheckCommand.java
(original)
+++
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/DataStoreCheckCommand.java
Thu May 18 03:36:19 2017
@@ -305,4 +305,4 @@ public class DataStoreCheckCommand imple
System.out.println(count + " blob ids found");
System.out.println("Finished in " + watch.elapsed(TimeUnit.SECONDS) +
" seconds");
}
-}
\ No newline at end of file
+}
Modified:
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/Utils.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/Utils.java?rev=1795475&r1=1795474&r2=1795475&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/Utils.java
(original)
+++
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/Utils.java
Thu May 18 03:36:19 2017
@@ -233,7 +233,9 @@ class Utils {
String cfgPath = s3dsConfig.value(options);
Properties props = loadAndTransformProps(cfgPath);
s3ds.setProperties(props);
- s3ds.init(null);
+ File homeDir = Files.createTempDir();
+ closer.register(asCloseable(homeDir));
+ s3ds.init(homeDir.getAbsolutePath());
delegate = s3ds;
} else if (options.has(azureBlobDSConfig)) {
AzureDataStore azureds = new AzureDataStore();
@@ -312,4 +314,4 @@ class Utils {
}
return props;
}
-}
\ No newline at end of file
+}
Modified:
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/cli/BlobStoreFixtureProvider.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/cli/BlobStoreFixtureProvider.java?rev=1795475&r1=1795474&r2=1795475&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/cli/BlobStoreFixtureProvider.java
(original)
+++
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/cli/BlobStoreFixtureProvider.java
Thu May 18 03:36:19 2017
@@ -68,7 +68,9 @@ public class BlobStoreFixtureProvider {
SharedS3DataStore s3ds = new SharedS3DataStore();
Properties props = loadAndTransformProps(bsopts.getS3ConfigPath());
s3ds.setProperties(props);
- s3ds.init(null);
+ File homeDir = Files.createTempDir();
+ closer.register(asCloseable(homeDir));
+ s3ds.init(homeDir.getAbsolutePath());
delegate = s3ds;
} else if(bsType == Type.AZURE){
AzureDataStore azureds = new AzureDataStore();
Modified:
jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/run/DataStoreCheckTest.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/run/DataStoreCheckTest.java?rev=1795475&r1=1795474&r2=1795475&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/run/DataStoreCheckTest.java
(original)
+++
jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/run/DataStoreCheckTest.java
Thu May 18 03:36:19 2017
@@ -21,7 +21,6 @@ package org.apache.jackrabbit.oak.run;
import static com.google.common.base.Charsets.UTF_8;
import static org.junit.Assert.assertEquals;
-import java.io.BufferedWriter;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.File;
@@ -34,6 +33,7 @@ import java.io.PrintStream;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
+import java.util.Properties;
import java.util.Random;
import java.util.Set;
@@ -41,9 +41,12 @@ import com.google.common.collect.Immutab
import com.google.common.collect.Iterables;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
-import com.google.common.io.Files;
+import org.apache.commons.io.FileUtils;
import org.apache.commons.io.filefilter.FileFilterUtils;
-import org.apache.commons.lang3.StringEscapeUtils;
+import org.apache.felix.cm.file.ConfigurationHandler;
+import org.apache.jackrabbit.core.data.DataStore;
+import
org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureDataStoreUtils;
+import org.apache.jackrabbit.oak.blob.cloud.s3.S3DataStoreUtils;
import org.apache.jackrabbit.oak.commons.FileIOUtils;
import org.apache.jackrabbit.oak.plugins.blob.datastore.DataStoreBlobStore;
import org.apache.jackrabbit.oak.plugins.blob.datastore.OakFileDataStore;
@@ -81,18 +84,48 @@ public class DataStoreCheckTest {
private String dsPath;
+ private DataStoreBlobStore setupDataStore;
+
+ private String dsOption;
+
@Before
public void setup() throws Exception {
- OakFileDataStore delegate = new OakFileDataStore();
- dsPath = temporaryFolder.newFolder().getAbsolutePath();
- delegate.setPath(dsPath);
- delegate.init(null);
- DataStoreBlobStore blobStore = new DataStoreBlobStore(delegate);
+ if (S3DataStoreUtils.isS3Configured()) {
+ Properties props = S3DataStoreUtils.getS3Config();
+ props.setProperty("cacheSize", "0");
+ DataStore ds =
S3DataStoreUtils.getS3DataStore(S3DataStoreUtils.getFixtures().get(0),
+ props,
+ temporaryFolder.newFolder().getAbsolutePath());
+ setupDataStore = new DataStoreBlobStore(ds);
+ cfgFilePath = createTempConfig(temporaryFolder.newFile(), props);
+ dsOption = "s3ds";
+ } else if (AzureDataStoreUtils.isAzureConfigured()) {
+ Properties props = AzureDataStoreUtils.getAzureConfig();
+ props.setProperty("cacheSize", "0");
+ DataStore ds = AzureDataStoreUtils.getAzureDataStore(props,
+ temporaryFolder.newFolder().getAbsolutePath());
+ setupDataStore = new DataStoreBlobStore(ds);
+ cfgFilePath = createTempConfig(temporaryFolder.newFile(), props);
+ dsOption = "azureblobds";
+ }
+ else {
+ OakFileDataStore delegate = new OakFileDataStore();
+ dsPath = temporaryFolder.newFolder().getAbsolutePath();
+ delegate.setPath(dsPath);
+ delegate.init(null);
+ setupDataStore = new DataStoreBlobStore(delegate);
+
+ File cfgFile = temporaryFolder.newFile();
+ Properties props = new Properties();
+ props.put("path", dsPath);
+ cfgFilePath = createTempConfig(cfgFile, props);
+ dsOption = "fds";
+ }
File storeFile = temporaryFolder.newFolder();
storePath = storeFile.getAbsolutePath();
FileStore fileStore = FileStoreBuilder.fileStoreBuilder(storeFile)
- .withBlobStore(blobStore)
+ .withBlobStore(setupDataStore)
.withMaxFileSize(256)
.withSegmentCacheSize(64)
.build();
@@ -104,7 +137,7 @@ public class DataStoreCheckTest {
blobsAdded = Sets.newHashSet();
for (int i = 0; i < numBlobs; i++) {
SegmentBlob b = (SegmentBlob) store.createBlob(randomStream(i,
18342));
- Iterator<String> idIter = blobStore.resolveChunks(b.getBlobId());
+ Iterator<String> idIter =
setupDataStore.resolveChunks(b.getBlobId());
while (idIter.hasNext()) {
String chunk = idIter.next();
blobsAdded.add(chunk);
@@ -115,14 +148,7 @@ public class DataStoreCheckTest {
store.merge(a, EmptyHook.INSTANCE, CommitInfo.EMPTY);
log.info("Created blobs : {}", blobsAdded);
- File cfgFile = temporaryFolder.newFile();
- BufferedWriter writer = Files.newWriter(cfgFile, UTF_8);
- FileIOUtils.writeAsLine(writer, "path=\"" +
StringEscapeUtils.escapeJava(dsPath) + "\"",false);
- writer.close();
- cfgFilePath = cfgFile.getAbsolutePath();
-
fileStore.close();
- blobStore.close();
}
@After
@@ -133,6 +159,7 @@ public class DataStoreCheckTest {
@Test
public void testCorrect() throws Exception {
File dump = temporaryFolder.newFolder();
+ setupDataStore.close();
testAllParams(dump);
}
@@ -140,17 +167,12 @@ public class DataStoreCheckTest {
public void testConsistency() throws Exception {
File dump = temporaryFolder.newFolder();
- // Delete a random blob from datastore
- OakFileDataStore delegate = new OakFileDataStore();
- delegate.setPath(dsPath);
- delegate.init(null);
- DataStoreBlobStore blobStore = new DataStoreBlobStore(delegate);
-
Random rand = new Random();
String deletedBlobId = Iterables.get(blobsAdded,
rand.nextInt(blobsAdded.size()));
blobsAdded.remove(deletedBlobId);
- long count =
blobStore.countDeleteChunks(ImmutableList.of(deletedBlobId), 0);
+ long count =
setupDataStore.countDeleteChunks(ImmutableList.of(deletedBlobId), 0);
assertEquals(1, count);
+ setupDataStore.close();
testAllParams(dump);
@@ -159,10 +181,10 @@ public class DataStoreCheckTest {
assertFileEquals(dump, "[consistency]",
Sets.newHashSet(deletedBlobId));
}
- public void testAllParams(File dump) throws Exception {
+ private void testAllParams(File dump) throws Exception {
DataStoreCheckCommand checkCommand = new DataStoreCheckCommand();
List<String> argsList = Lists
- .newArrayList("--id", "--ref", "--consistency", "--fds",
cfgFilePath, "--store", storePath,
+ .newArrayList("--id", "--ref", "--consistency", "--" + dsOption,
cfgFilePath, "--store", storePath,
"--dump", dump.getAbsolutePath());
checkCommand.execute(argsList.toArray(new String[0]));
@@ -170,9 +192,10 @@ public class DataStoreCheckTest {
@Test
public void testMissingOpParams() throws Exception {
+ setupDataStore.close();
File dump = temporaryFolder.newFolder();
List<String> argsList = Lists
- .newArrayList("--fds", cfgFilePath, "--store", storePath,
+ .newArrayList("--" + dsOption, cfgFilePath, "--store", storePath,
"--dump", dump.getAbsolutePath());
log.info("Running testMissinOpParams: {}", argsList);
testIncorrectParams(argsList, Lists.newArrayList("Missing required
option(s)", "'id'", "'ref'", "'consistency'"));
@@ -180,6 +203,7 @@ public class DataStoreCheckTest {
@Test
public void testTarNoDS() throws Exception {
+ setupDataStore.close();
File dump = temporaryFolder.newFolder();
List<String> argsList = Lists
.newArrayList("--id", "--ref", "--consistency", "--store",
storePath,
@@ -190,14 +214,15 @@ public class DataStoreCheckTest {
@Test
public void testOpNoStore() throws Exception {
+ setupDataStore.close();
File dump = temporaryFolder.newFolder();
List<String> argsList = Lists
- .newArrayList("--consistency", "--fds", cfgFilePath,
+ .newArrayList("--consistency", "--" + dsOption, cfgFilePath,
"--dump", dump.getAbsolutePath());
testIncorrectParams(argsList, Lists.newArrayList("Missing required
option(s) ['store']"));
argsList = Lists
- .newArrayList("--ref", "--fds", cfgFilePath,
+ .newArrayList("--ref", "--" + dsOption, cfgFilePath,
"--dump", dump.getAbsolutePath());
testIncorrectParams(argsList, Lists.newArrayList("Missing required
option(s) ['store']"));
}
@@ -236,4 +261,10 @@ public class DataStoreCheckTest {
r.nextBytes(data);
return new ByteArrayInputStream(data);
}
+
+ private static String createTempConfig(File cfgFile, Properties props)
throws IOException {
+ FileOutputStream fos = FileUtils.openOutputStream(cfgFile);
+ ConfigurationHandler.write(fos, props);
+ return cfgFile.getAbsolutePath();
+ }
}
