Author: bfoster
Date: Mon Mar 24 08:26:48 2014
New Revision: 1580783
URL: http://svn.apache.org/r1580783
Log:
- Updates to Amazon S3 DataTransferer
Added:
oodt/trunk/filemgr/src/test/org/apache/oodt/cas/filemgr/datatransfer/TestS3DataTransfererFactory.java
(with props)
Modified:
oodt/trunk/filemgr/pom.xml
oodt/trunk/filemgr/src/main/bin/filemgr
oodt/trunk/filemgr/src/main/bin/filemgr-client
oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/S3DataTransferer.java
oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/S3DataTransfererFactory.java
oodt/trunk/filemgr/src/main/resources/filemgr.properties
Modified: oodt/trunk/filemgr/pom.xml
URL:
http://svn.apache.org/viewvc/oodt/trunk/filemgr/pom.xml?rev=1580783&r1=1580782&r2=1580783&view=diff
==============================================================================
--- oodt/trunk/filemgr/pom.xml (original)
+++ oodt/trunk/filemgr/pom.xml Mon Mar 24 08:26:48 2014
@@ -282,7 +282,7 @@
<dependency>
<groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk</artifactId>
- <version>1.5.7</version>
+ <version>1.7.4</version>
</dependency>
<dependency>
<groupId>junit</groupId>
Modified: oodt/trunk/filemgr/src/main/bin/filemgr
URL:
http://svn.apache.org/viewvc/oodt/trunk/filemgr/src/main/bin/filemgr?rev=1580783&r1=1580782&r2=1580783&view=diff
==============================================================================
--- oodt/trunk/filemgr/src/main/bin/filemgr (original)
+++ oodt/trunk/filemgr/src/main/bin/filemgr Mon Mar 24 08:26:48 2014
@@ -48,12 +48,16 @@ export PATH
## just to be on the safe side
mkdir -p ${RUN_HOME}
+for file in `find ../lib/*.jar`; do
+ LIB_DEPS="${file}:${LIB_DEPS}"
+done
+
# See how we were called.
case "$1" in
start)
echo -n "Starting cas file manager: "
$JAVA_HOME/bin/java \
- -Djava.ext.dirs=${CAS_FILEMGR_HOME}/lib \
+ -cp ${LIB_DEPS} \
-Djava.util.logging.config.file=${CAS_FILEMGR_HOME}/etc/logging.properties \
-Dorg.apache.oodt.cas.filemgr.properties=${CAS_FILEMGR_PROPS} \
org.apache.oodt.cas.filemgr.system.XmlRpcFileManager \
Modified: oodt/trunk/filemgr/src/main/bin/filemgr-client
URL:
http://svn.apache.org/viewvc/oodt/trunk/filemgr/src/main/bin/filemgr-client?rev=1580783&r1=1580782&r2=1580783&view=diff
==============================================================================
--- oodt/trunk/filemgr/src/main/bin/filemgr-client (original)
+++ oodt/trunk/filemgr/src/main/bin/filemgr-client Mon Mar 24 08:26:48 2014
@@ -24,10 +24,14 @@ fi
export JAVA_HOME
+for file in `find ../lib/*.jar`; do
+ LIB_DEPS="${file}:${LIB_DEPS}"
+done
+
$JAVA_HOME/bin/java \
- -Djava.ext.dirs=../lib \
+ -cp ${LIB_DEPS} \
-Dorg.apache.oodt.cas.filemgr.properties=../etc/filemgr.properties \
-Djava.util.logging.config.file=../etc/logging.properties \
-
-Dorg.apache.oodt.cas.cli.action.spring.config=../policy/cmd-line-actions.xml \
-
-Dorg.apache.oodt.cas.cli.option.spring.config=../policy/cmd-line-options.xml \
+
-Dorg.apache.oodt.cas.cli.action.spring.config=../policy/cmd-line-actions.xml \
+
-Dorg.apache.oodt.cas.cli.option.spring.config=../policy/cmd-line-options.xml \
org.apache.oodt.cas.filemgr.system.XmlRpcFileManagerClient $*
Modified:
oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/S3DataTransferer.java
URL:
http://svn.apache.org/viewvc/oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/S3DataTransferer.java?rev=1580783&r1=1580782&r2=1580783&view=diff
==============================================================================
---
oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/S3DataTransferer.java
(original)
+++
oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/S3DataTransferer.java
Mon Mar 24 08:26:48 2014
@@ -21,6 +21,8 @@ import static com.google.common.base.Pre
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
import java.net.URL;
import org.apache.oodt.cas.filemgr.structs.Product;
@@ -56,13 +58,14 @@ public class S3DataTransferer implements
@Override
public void transferProduct(Product product) throws
DataTransferException, IOException {
for (Reference ref : product.getProductReferences()) {
+ String origRef = stripProtocol(ref.getOrigReference(), false);
+ String dataStoreRef =
stripProtocol(ref.getDataStoreReference(), true);
try {
- s3Client.putObject(new
PutObjectRequest(bucketName, ref.getDataStoreReference(), new File(ref
- .getOrigReference())));
+ s3Client.putObject(new
PutObjectRequest(bucketName, dataStoreRef, new File(origRef)));
} catch (AmazonClientException e) {
throw new DataTransferException(String.format(
- "Failed to upload product reference %s to
S3 at %s", ref.getOrigReference(),
- ref.getDataStoreReference()), e);
+ "Failed to upload product reference %s to
S3 at %s", origRef,
+ dataStoreRef), e);
}
}
}
@@ -71,7 +74,7 @@ public class S3DataTransferer implements
public void retrieveProduct(Product product, File directory) throws
DataTransferException,
IOException {
for (Reference ref : product.getProductReferences()) {
- GetObjectRequest request = new
GetObjectRequest(bucketName, ref.getDataStoreReference());
+ GetObjectRequest request = new
GetObjectRequest(bucketName, stripProtocol(ref.getDataStoreReference(), true));
S3Object file = s3Client.getObject(request);
stageFile(file, ref, directory);
}
@@ -83,7 +86,7 @@ public class S3DataTransferer implements
try {
inStream = file.getObjectContent();
outStream = new FileOutputStream(new File(directory,
new File(
- ref.getDataStoreReference()).getName()));
+ stripProtocol(ref.getDataStoreReference(),
false)).getName()));
IOUtils.copy(inStream, outStream);
} catch (IOException e) {
throw e;
@@ -92,4 +95,16 @@ public class S3DataTransferer implements
try { outStream.close(); } catch (Exception e) {}
}
}
+
+ private String stripProtocol(String ref, boolean stripLeadingSeparator)
throws IOException {
+ try {
+ String path = new URI(ref).getPath();
+ if (stripLeadingSeparator) {
+ path = path.replaceAll("^/", "");
+ }
+ return path;
+ } catch (URISyntaxException e) {
+ throw new IOException(e);
+ }
+ }
}
Modified:
oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/S3DataTransfererFactory.java
URL:
http://svn.apache.org/viewvc/oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/S3DataTransfererFactory.java?rev=1580783&r1=1580782&r2=1580783&view=diff
==============================================================================
---
oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/S3DataTransfererFactory.java
(original)
+++
oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/S3DataTransfererFactory.java
Mon Mar 24 08:26:48 2014
@@ -19,7 +19,6 @@ package org.apache.oodt.cas.filemgr.data
import com.amazonaws.auth.BasicAWSCredentials;
import com.amazonaws.regions.Region;
import com.amazonaws.regions.Regions;
-import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3Client;
/**
@@ -39,15 +38,15 @@ public class S3DataTransfererFactory imp
"org.apache.oodt.cas.filemgr.datatransfer.s3.secret.key";
@Override
- public DataTransfer createDataTransfer() {
+ public S3DataTransferer createDataTransfer() {
String bucketName = System.getProperty(BUCKET_NAME_PROPERTY);
- String region = System.getProperty(REGION_PROPERTY);
+ String region = System.getProperty(REGION_PROPERTY);
String accessKey = System.getProperty(ACCESS_KEY_PROPERTY);
- String secretKey = System.getProperty(SECRET_KEY_PROPERTY);
+ String secretKey = System.getProperty(SECRET_KEY_PROPERTY);
- AmazonS3 s3 = new AmazonS3Client(new
BasicAWSCredentials(accessKey, secretKey));
+ AmazonS3Client s3 = new AmazonS3Client(new
BasicAWSCredentials(accessKey, secretKey));
s3.setRegion(Region.getRegion(Regions.valueOf(region)));
- return new S3DataTransferer(s3, bucketName);
+ return new S3DataTransferer(s3, bucketName);
}
}
Modified: oodt/trunk/filemgr/src/main/resources/filemgr.properties
URL:
http://svn.apache.org/viewvc/oodt/trunk/filemgr/src/main/resources/filemgr.properties?rev=1580783&r1=1580782&r2=1580783&view=diff
==============================================================================
--- oodt/trunk/filemgr/src/main/resources/filemgr.properties (original)
+++ oodt/trunk/filemgr/src/main/resources/filemgr.properties Mon Mar 24
08:26:48 2014
@@ -118,7 +118,7 @@ org.apache.oodt.cas.filemgr.datatransfer
# Region can be any of the values found here:
#
http://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/regions/Regions.html
org.apache.oodt.cas.filemgr.datatransfer.s3.bucket.name=some_bucket_name
-org.apache.oodt.cas.filemgr.datatransfer.s3.region=EU_WEST_1
+org.apache.oodt.cas.filemgr.datatransfer.s3.region=EU_WEST_1
org.apache.oodt.cas.filemgr.datatransfer.s3.access.key=s3_access_key
org.apache.oodt.cas.filemgr.datatransfer.s3.secret.key=s3_secret_key
Added:
oodt/trunk/filemgr/src/test/org/apache/oodt/cas/filemgr/datatransfer/TestS3DataTransfererFactory.java
URL:
http://svn.apache.org/viewvc/oodt/trunk/filemgr/src/test/org/apache/oodt/cas/filemgr/datatransfer/TestS3DataTransfererFactory.java?rev=1580783&view=auto
==============================================================================
---
oodt/trunk/filemgr/src/test/org/apache/oodt/cas/filemgr/datatransfer/TestS3DataTransfererFactory.java
(added)
+++
oodt/trunk/filemgr/src/test/org/apache/oodt/cas/filemgr/datatransfer/TestS3DataTransfererFactory.java
Mon Mar 24 08:26:48 2014
@@ -0,0 +1,41 @@
+package org.apache.oodt.cas.filemgr.datatransfer;
+
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.not;
+import static org.hamcrest.Matchers.nullValue;
+import static org.junit.Assert.assertThat;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+/**
+ * Test class for {@link S3DataTransfererFactory}.
+ *
+ * @author [email protected] (Brian Foster)
+ */
+@RunWith(JUnit4.class)
+public class TestS3DataTransfererFactory {
+
+ private static final String BUCKET_NAME_PROPERTY =
+ "org.apache.oodt.cas.filemgr.datatransfer.s3.bucket.name";
+ private static final String REGION_PROPERTY =
+ "org.apache.oodt.cas.filemgr.datatransfer.s3.region";
+ private static final String ACCESS_KEY_PROPERTY =
+ "org.apache.oodt.cas.filemgr.datatransfer.s3.access.key";
+ private static final String SECRET_KEY_PROPERTY =
+ "org.apache.oodt.cas.filemgr.datatransfer.s3.secret.key";
+
+ @Test
+ public void testCreateDataTransferer() {
+ System.setProperty(BUCKET_NAME_PROPERTY, "test-bucket");
+ System.setProperty(REGION_PROPERTY, "US_WEST_1");
+ System.setProperty(ACCESS_KEY_PROPERTY, "23123123123");
+ System.setProperty(SECRET_KEY_PROPERTY, "00101010101");
+
+ S3DataTransfererFactory factory = new S3DataTransfererFactory();
+ S3DataTransferer transferer = factory.createDataTransfer();
+
+ assertThat(transferer, is(not(nullValue())));
+ }
+}
Propchange:
oodt/trunk/filemgr/src/test/org/apache/oodt/cas/filemgr/datatransfer/TestS3DataTransfererFactory.java
------------------------------------------------------------------------------
svn:mime-type = text/plain
