Correctly writes export file to s3 using file stream buffers.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/15db8f4c Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/15db8f4c Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/15db8f4c Branch: refs/pull/70/merge Commit: 15db8f4cf5864ef0530071ed37883bac44f4a282 Parents: 68bd76b Author: George Reyes <[email protected]> Authored: Wed Feb 12 17:09:14 2014 -0800 Committer: George Reyes <[email protected]> Committed: Wed Feb 12 17:09:14 2014 -0800 ---------------------------------------------------------------------- .../management/export/ExportServiceImpl.java | 54 ++++++++++++-------- .../usergrid/management/export/S3Export.java | 4 +- .../management/export/S3ExportImpl.java | 13 +++-- 3 files changed, 45 insertions(+), 26 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/15db8f4c/stack/services/src/main/java/org/usergrid/management/export/ExportServiceImpl.java ---------------------------------------------------------------------- diff --git a/stack/services/src/main/java/org/usergrid/management/export/ExportServiceImpl.java b/stack/services/src/main/java/org/usergrid/management/export/ExportServiceImpl.java index c40f7f1..099f23e 100644 --- a/stack/services/src/main/java/org/usergrid/management/export/ExportServiceImpl.java +++ b/stack/services/src/main/java/org/usergrid/management/export/ExportServiceImpl.java @@ -1,9 +1,12 @@ package org.usergrid.management.export; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; import java.io.File; import java.io.IOException; -import java.io.PrintWriter; +import java.io.InputStream; +import java.io.ObjectOutputStream; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -194,9 +197,14 @@ public class ExportServiceImpl implements ExportService{ return jobUUID; } - +//TODO: make multipart streaming functional + //currently only stores the collection in memory then flushes it. private void exportApplicationsForOrg( Map.Entry<UUID, String> organization,final ExportInfo config ) throws Exception { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + ObjectOutputStream oos = new ObjectOutputStream(baos); + + logger.info( "" + organization ); @@ -219,7 +227,7 @@ public class ExportServiceImpl implements ExportService{ String appFileName = prepareOutputFileName( "application", application.getValue() ); - JsonGenerator jg = getJsonGenerator( new File( appFileName ) ); + JsonGenerator jg = getJsonGenerator( oos ); // load the dictionary @@ -258,16 +266,17 @@ public class ExportServiceImpl implements ExportService{ nsEntity.setMetadata( "collections", collections ); jg.writeStartArray(); - jg.writeObject( nsEntity ); + //jg.writeObject( nsEntity ); // Create a GENERATOR for the application collections. //JsonGenerator collectionsJg = getJsonGenerator( createOutputFile( "collections", application.getValue() ) ); - String collectionsFilename = prepareOutputFileName( "collections","appDummyName" ); - JsonGenerator collectionsJg = getJsonGenerator( new File( collectionsFilename ) ); + //String collectionsFilename = prepareOutputFileName( "collections","appDummyName" ); + //JsonGenerator collectionsJg = getJsonGenerator( oos ); - collectionsJg.writeStartObject(); + //collectionsJg.writeStartObject(); + // jg.writeStartObject(); Map<String, Object> metadata = em.getApplicationCollectionMetadata(); //don't need to echo as not a command line tool anymore //echo( JsonUtils.mapToFormattedJsonString( metadata ) ); @@ -287,10 +296,10 @@ public class ExportServiceImpl implements ExportService{ for ( Entity entity : entities ) { // Export the entity first and later the collections for // this entity. - jg.writeObject( entity ); + //jg.writeObject( entity ); //echo( entity ); - saveCollectionMembers( collectionsJg, em, application.getValue(), entity ); + saveCollectionMembers( jg, em, application.getValue(), entity ); } //we're done @@ -306,15 +315,20 @@ public class ExportServiceImpl implements ExportService{ } // Close writer for the collections for this application. - collectionsJg.writeEndObject(); - collectionsJg.close(); + // collectionsJg.writeEndObject(); + // collectionsJg.close(); // Close writer and file for this application. + + // logger.warn(); jg.writeEndArray(); jg.close(); - s3Export.copyToS3( appFileName, config ); + oos.flush(); + oos.close(); + + InputStream is = new ByteArrayInputStream( baos.toByteArray() ); + s3Export.copyToS3( is, config ); //below line doesn't copy very good data anyways. - //copyToS3( collectionsFilename, config ); } } @@ -335,7 +349,7 @@ public class ExportServiceImpl implements ExportService{ return; } - jg.writeFieldName( entity.getUuid().toString() ); + // jg.writeFieldName( entity.getUuid().toString() ); jg.writeStartObject(); for ( String collectionName : collections ) { @@ -368,15 +382,15 @@ public class ExportServiceImpl implements ExportService{ jg.writeEndObject(); } - protected JsonGenerator getJsonGenerator( String outFile ) throws IOException { - return getJsonGenerator( new File( outputDir, outFile ) ); - } + // protected JsonGenerator getJsonGenerator( String outFile ) throws IOException { + // return getJsonGenerator( new File( outputDir, outFile ) ); + // } - protected JsonGenerator getJsonGenerator( File outFile ) throws IOException { + protected JsonGenerator getJsonGenerator( ObjectOutputStream oos ) throws IOException { //TODO:shouldn't the below be UTF-16? - PrintWriter out = new PrintWriter( outFile, "UTF-8" ); - JsonGenerator jg = jsonFactory.createJsonGenerator( out ); + //PrintWriter out = new PrintWriter( outFile, "UTF-8" ); + JsonGenerator jg = jsonFactory.createJsonGenerator( oos ); jg.setPrettyPrinter( new DefaultPrettyPrinter() ); jg.setCodec( new ObjectMapper() ); return jg; http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/15db8f4c/stack/services/src/main/java/org/usergrid/management/export/S3Export.java ---------------------------------------------------------------------- diff --git a/stack/services/src/main/java/org/usergrid/management/export/S3Export.java b/stack/services/src/main/java/org/usergrid/management/export/S3Export.java index 6895da5..cc4486c 100644 --- a/stack/services/src/main/java/org/usergrid/management/export/S3Export.java +++ b/stack/services/src/main/java/org/usergrid/management/export/S3Export.java @@ -1,6 +1,8 @@ package org.usergrid.management.export; +import java.io.InputStream; + import org.usergrid.management.ExportInfo; @@ -9,6 +11,6 @@ import org.usergrid.management.ExportInfo; * */ public interface S3Export { - void copyToS3( String fileName, ExportInfo exportInfo ); + void copyToS3( InputStream inputStream, ExportInfo exportInfo ); } http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/15db8f4c/stack/services/src/main/java/org/usergrid/management/export/S3ExportImpl.java ---------------------------------------------------------------------- diff --git a/stack/services/src/main/java/org/usergrid/management/export/S3ExportImpl.java b/stack/services/src/main/java/org/usergrid/management/export/S3ExportImpl.java index fc8e281..52afb71 100644 --- a/stack/services/src/main/java/org/usergrid/management/export/S3ExportImpl.java +++ b/stack/services/src/main/java/org/usergrid/management/export/S3ExportImpl.java @@ -1,7 +1,7 @@ package org.usergrid.management.export; -import java.io.File; +import java.io.InputStream; import java.util.Properties; import org.jclouds.ContextBuilder; @@ -27,8 +27,9 @@ import com.google.inject.Module; * */ public class S3ExportImpl implements S3Export { + @Override - public void copyToS3( String fileName, final ExportInfo exportInfo ) { + public void copyToS3( final InputStream inputStream, final ExportInfo exportInfo ) { Logger logger = LoggerFactory.getLogger( ExportServiceImpl.class ); /*won't need any of the properties as I have the export info*/ @@ -63,11 +64,12 @@ public class S3ExportImpl implements S3Export { } try { - File file = new File( fileName ); + + AsyncBlobStore blobStore = context.getAsyncBlobStore(); BlobBuilder blobBuilder = - blobStore.blobBuilder( file.getName() ).payload( file ).calculateMD5().contentType( "text/plain" ) - .contentLength( file.length() ); + blobStore.blobBuilder( "test.json" ).payload( inputStream ).calculateMD5().contentType( "text/plain" ); + Blob blob = blobBuilder.build(); @@ -78,6 +80,7 @@ public class S3ExportImpl implements S3Export { catch ( Exception e ) { logger.error( "Error uploading to blob store", e ); } + } }
