Repository: incubator-usergrid Updated Branches: refs/heads/USERGRID-579-jcloud171 [created] e0d9fe675 Updated Tags: refs/tags/2015-04-21-ssofix [created] 4960eda96 refs/tags/2015-04-28-ssotest [created] 76d3d0020 refs/tags/2015-04-29-ssotest [created] b4715c043 refs/tags/20150505-dedicated-sso [created] c1e613f8c refs/tags/20150512-sso-conn-pool [created] a663f0761 refs/tags/20150515-jclouds-fix [created] a40180581
Move to JClouds 1.7.1 for AWS S3 access to work around problems in OpenJDK 1.7 and Oracle JDK 8. Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/e707628b Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/e707628b Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/e707628b Branch: refs/heads/USERGRID-579-jcloud171 Commit: e707628bf3a0aba3975717e47fab7951e58ae264 Parents: 5f4a66b Author: Dave Johnson <[email protected]> Authored: Fri May 15 12:16:51 2015 -0400 Committer: Dave Johnson <[email protected]> Committed: Fri May 15 12:16:51 2015 -0400 ---------------------------------------------------------------------- stack/pom.xml | 2 +- .../rest/applications/ServiceResource.java | 2 +- .../applications/assets/AssetResourceIT.java | 50 ++++++++++++---- .../services/assets/data/S3BinaryStore.java | 63 ++++++++++++-------- 4 files changed, 79 insertions(+), 38 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e707628b/stack/pom.xml ---------------------------------------------------------------------- diff --git a/stack/pom.xml b/stack/pom.xml index 4f50779..5c3d4fe 100644 --- a/stack/pom.xml +++ b/stack/pom.xml @@ -95,7 +95,7 @@ <hector-version>1.1-4</hector-version> <hector-test-version>1.1-4</hector-test-version> <jackson-version>1.9.9</jackson-version> - <jclouds.version>1.6.2-incubating</jclouds.version> + <jclouds.version>1.7.1</jclouds.version> <jersey-version>1.18</jersey-version> <junit-version>4.11</junit-version> <log4j-version>1.2.16</log4j-version> http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e707628b/stack/rest/src/main/java/org/apache/usergrid/rest/applications/ServiceResource.java ---------------------------------------------------------------------- diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/applications/ServiceResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/applications/ServiceResource.java index d5ef311..3eee96b 100644 --- a/stack/rest/src/main/java/org/apache/usergrid/rest/applications/ServiceResource.java +++ b/stack/rest/src/main/java/org/apache/usergrid/rest/applications/ServiceResource.java @@ -595,7 +595,7 @@ public class ServiceResource extends AbstractContextResource { ServiceResults serviceResults = executeServiceRequest( ui, response, ServiceAction.GET, null ); Entity entity = serviceResults.getEntity(); - LOG.info( "In AssetsResource.findAsset with id: {}, range: {}, modifiedSince: {}", + LOG.info( "In ServiceResource.executeStreamGet with id: {}, range: {}, modifiedSince: {}", new Object[] { entityId, rangeHeader, modifiedSince } ); Map<String, Object> fileMetadata = AssetUtils.getFileMetadata( entity ); http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e707628b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/assets/AssetResourceIT.java ---------------------------------------------------------------------- diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/assets/AssetResourceIT.java b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/assets/AssetResourceIT.java index b41659d..666f95e 100644 --- a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/assets/AssetResourceIT.java +++ b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/assets/AssetResourceIT.java @@ -139,10 +139,14 @@ public class AssetResourceIT extends AbstractRestIT { UserRepo.INSTANCE.load( resource(), access_token ); byte[] data = IOUtils.toByteArray( this.getClass().getResourceAsStream( "/file-bigger-than-5M" ) ); - FormDataMultiPart form = new FormDataMultiPart().field( "file", data, MediaType.MULTIPART_FORM_DATA_TYPE ); - JsonNode node = resource().path( "/test-organization/test-app/foos" ).queryParam( "access_token", access_token ) - .accept( MediaType.APPLICATION_JSON ).type( MediaType.MULTIPART_FORM_DATA ) + FormDataMultiPart form = new FormDataMultiPart() + .field( "file", data, MediaType.MULTIPART_FORM_DATA_TYPE ); + + JsonNode node = resource().path( "/test-organization/test-app/foos" ) + .queryParam( "access_token", access_token ) + .accept( MediaType.APPLICATION_JSON ) + .type( MediaType.MULTIPART_FORM_DATA ) .post( JsonNode.class, form ); JsonNode idNode = node.get( "entities" ).get( 0 ).get( "uuid" ); @@ -151,25 +155,47 @@ public class AssetResourceIT extends AbstractRestIT { logNode( node ); // get entity - node = resource().path( "/test-organization/test-app/foos/" + uuid ).queryParam( "access_token", access_token ) - .accept( MediaType.APPLICATION_JSON_TYPE ).get( JsonNode.class ); + node = resource().path( "/test-organization/test-app/foos/" + uuid ) + .queryParam( "access_token", access_token ) + .accept( MediaType.APPLICATION_JSON_TYPE ) + .get( JsonNode.class ); + logNode( node ); assertEquals( "application/octet-stream", node.findValue( AssetUtils.CONTENT_TYPE ).getTextValue() ); assertEquals( 5324800, node.findValue( AssetUtils.CONTENT_LENGTH ).getIntValue() ); idNode = node.get( "entities" ).get( 0 ).get( "uuid" ); assertEquals( uuid, idNode.getTextValue() ); - // get data - InputStream is = - resource().path( "/test-organization/test-app/foos/" + uuid ).queryParam( "access_token", access_token ) - .accept( MediaType.APPLICATION_OCTET_STREAM_TYPE ).get( InputStream.class ); + int retries = 0; + boolean done = false; + byte[] foundData = new byte[0]; + + // retry until upload complete + while ( !done && retries < 30 ) { + + // get data + try { + InputStream is = resource().path( "/test-organization/test-app/foos/" + uuid ) + .queryParam( "access_token", access_token ) + .accept( MediaType.APPLICATION_OCTET_STREAM_TYPE ) + .get( InputStream.class ); + + foundData = IOUtils.toByteArray( is ); + done = true; + + } catch ( Exception intentiallyIgnored ) {} + + Thread.sleep(1000); + retries++; + } - byte[] foundData = IOUtils.toByteArray( is ); assertEquals( 5324800, foundData.length ); // delete - node = resource().path( "/test-organization/test-app/foos/" + uuid ).queryParam( "access_token", access_token ) - .accept( MediaType.APPLICATION_JSON_TYPE ).delete( JsonNode.class ); + node = resource().path( "/test-organization/test-app/foos/" + uuid ) + .queryParam( "access_token", access_token ) + .accept( MediaType.APPLICATION_JSON_TYPE ) + .delete( JsonNode.class ); } http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e707628b/stack/services/src/main/java/org/apache/usergrid/services/assets/data/S3BinaryStore.java ---------------------------------------------------------------------- diff --git a/stack/services/src/main/java/org/apache/usergrid/services/assets/data/S3BinaryStore.java b/stack/services/src/main/java/org/apache/usergrid/services/assets/data/S3BinaryStore.java index e1748d3..29b5e47 100644 --- a/stack/services/src/main/java/org/apache/usergrid/services/assets/data/S3BinaryStore.java +++ b/stack/services/src/main/java/org/apache/usergrid/services/assets/data/S3BinaryStore.java @@ -104,7 +104,7 @@ public class S3BinaryStore implements BinaryStore { @Override public void write( final UUID appId, final Entity entity, InputStream inputStream ) throws IOException { - String uploadFileName = AssetUtils.buildAssetKey( appId, entity ); + final String uploadFileName = AssetUtils.buildAssetKey( appId, entity ); ByteArrayOutputStream baos = new ByteArrayOutputStream(); long written = IOUtils.copyLarge( inputStream, baos, 0, FIVE_MB ); byte[] data = baos.toByteArray(); @@ -112,13 +112,13 @@ public class S3BinaryStore implements BinaryStore { final Map<String, Object> fileMetadata = AssetUtils.getFileMetadata( entity ); fileMetadata.put( AssetUtils.LAST_MODIFIED, System.currentTimeMillis() ); - String mimeType = AssetMimeHandler.get().getMimeType( entity, data ); + final String mimeType = AssetMimeHandler.get().getMimeType( entity, data ); if ( written < FIVE_MB ) { // total smaller than 5mb BlobStore blobStore = getContext().getBlobStore(); - BlobBuilder.PayloadBlobBuilder bb = - blobStore.blobBuilder( uploadFileName ).payload( data ).calculateMD5().contentType( mimeType ); + BlobBuilder.PayloadBlobBuilder bb = blobStore.blobBuilder( uploadFileName ) + .payload( data ).calculateMD5().contentType( mimeType ); fileMetadata.put( AssetUtils.CONTENT_LENGTH, written ); if ( fileMetadata.get( AssetUtils.CONTENT_DISPOSITION ) != null ) { @@ -134,10 +134,11 @@ public class S3BinaryStore implements BinaryStore { } else { // bigger than 5mb... dump 5 mb tmp files and upload from them - // todo: yes, AsyncBlobStore is deprecated, but there appears to be no replacement yet - final AsyncBlobStore blobStore = getContext().getAsyncBlobStore(); + // create temp file and copy entire file to that temp file - File tempFile = File.createTempFile( entity.getUuid().toString(), "tmp" ); + LOG.debug( "Writing temp file for S3 upload" ); + + final File tempFile = File.createTempFile( entity.getUuid().toString(), "tmp" ); tempFile.deleteOnExit(); OutputStream os = null; try { @@ -149,38 +150,52 @@ public class S3BinaryStore implements BinaryStore { IOUtils.closeQuietly( os ); } - BlobBuilder.PayloadBlobBuilder bb = - blobStore.blobBuilder( uploadFileName ).payload( tempFile ).calculateMD5().contentType( mimeType ); - fileMetadata.put( AssetUtils.CONTENT_LENGTH, written ); - if ( fileMetadata.get( AssetUtils.CONTENT_DISPOSITION ) != null ) { - bb.contentDisposition( fileMetadata.get( AssetUtils.CONTENT_DISPOSITION ).toString() ); - } - final Blob blob = bb.build(); - final File finalTempFile = tempFile; - final ListenableFuture<String> future = - blobStore.putBlob( bucketName, blob, PutOptions.Builder.multipart() ); + // JClouds no longer supports async blob store, so we have to do this fun stuff - Runnable listener = new Runnable() { + LOG.debug( "Starting upload thread" ); + + Thread uploadThread = new Thread( new Runnable() { @Override public void run() { try { - String eTag = future.get(); + LOG.debug( "S3 upload thread started" ); + + BlobStore blobStore = getContext().getBlobStore(); + + BlobBuilder.PayloadBlobBuilder bb = blobStore.blobBuilder( uploadFileName ) + .payload( tempFile ).calculateMD5().contentType( mimeType ); + + if ( fileMetadata.get( AssetUtils.CONTENT_DISPOSITION ) != null ) { + bb.contentDisposition( fileMetadata.get( AssetUtils.CONTENT_DISPOSITION ).toString() ); + } + final Blob blob = bb.build(); + + String md5sum = Hex.encodeHexString( blob.getMetadata().getContentMetadata().getContentMD5() ); + fileMetadata.put( AssetUtils.CHECKSUM, md5sum ); + + LOG.debug( "S3 upload starting" ); + + String eTag = blobStore.putBlob( bucketName, blob ); fileMetadata.put( AssetUtils.E_TAG, eTag ); + + LOG.debug( "S3 upload complete eTag=" + eTag); + EntityManager em = emf.getEntityManager( appId ); em.update( entity ); - finalTempFile.delete(); + tempFile.delete(); } catch ( Exception e ) { LOG.error( "error uploading", e ); } - if ( finalTempFile != null && finalTempFile.exists() ) { - finalTempFile.delete(); + if ( tempFile != null && tempFile.exists() ) { + tempFile.delete(); } } - }; - future.addListener( listener, executor ); + }); + + uploadThread.start(); } }
