mocked filename output. Still didnât fix maven testFileConnection test for export. cleaned up code to only look at one org. Should make export a lot more easy to read/debug.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/152c72d4 Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/152c72d4 Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/152c72d4 Branch: refs/pull/70/merge Commit: 152c72d4432f592a1fcc226ea4ac559e9dbfb56e Parents: 88a2a48 Author: grey <[email protected]> Authored: Thu Feb 27 17:23:03 2014 -0800 Committer: grey <[email protected]> Committed: Thu Feb 27 17:23:03 2014 -0800 ---------------------------------------------------------------------- .../applications/ApplicationResource.java | 2 + .../apache/usergrid/management/ExportInfo.java | 4 + .../management/export/ExportServiceImpl.java | 96 +++++++++++++++++--- .../usergrid/management/export/S3Export.java | 4 + .../management/export/S3ExportImpl.java | 7 ++ .../cassandra/ManagementServiceIT.java | 28 ++++-- .../management/cassandra/MockS3ExportImpl.java | 11 ++- 7 files changed, 130 insertions(+), 22 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/152c72d4/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/applications/ApplicationResource.java ---------------------------------------------------------------------- diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/applications/ApplicationResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/applications/ApplicationResource.java index 0e9d8ed..0ddf61b 100644 --- a/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/applications/ApplicationResource.java +++ b/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/applications/ApplicationResource.java @@ -228,8 +228,10 @@ public class ApplicationResource extends AbstractContextResource { try { //parse the json into some useful object (the config params) ExportInfo objEx = new ExportInfo( json ); + objEx.setOrganizationId( organization.getUuid() ); objEx.setApplicationId( applicationId ); objEx.setCollection( colExport ); + jobUUID = exportService.schedule( objEx ); uuidRet.put( "jobUUID", jobUUID.toString() ); } http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/152c72d4/stack/services/src/main/java/org/apache/usergrid/management/ExportInfo.java ---------------------------------------------------------------------- diff --git a/stack/services/src/main/java/org/apache/usergrid/management/ExportInfo.java b/stack/services/src/main/java/org/apache/usergrid/management/ExportInfo.java index b1be504..d3d0097 100644 --- a/stack/services/src/main/java/org/apache/usergrid/management/ExportInfo.java +++ b/stack/services/src/main/java/org/apache/usergrid/management/ExportInfo.java @@ -22,6 +22,7 @@ public class ExportInfo extends TypedEntity { private String bucket_location; private UUID applicationId; //TODO: include application when initializing export info private String collection; //TODO: include collection when initializing export info + private UUID organizationId; public ExportInfo( Map<String, Object> exportData ) { @@ -40,6 +41,7 @@ public class ExportInfo extends TypedEntity { return applicationId; } + public UUID getOrganizationId() { return organizationId; } public String getPath() { return path; @@ -74,4 +76,6 @@ public class ExportInfo extends TypedEntity { public void setApplicationId( UUID appId ) { applicationId = appId;} public void setCollection ( String colName) { collection = colName; } + + public void setOrganizationId ( UUID orgId ) { organizationId = orgId; } } http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/152c72d4/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java ---------------------------------------------------------------------- diff --git a/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java b/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java index b716293..7e5833f 100644 --- a/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java +++ b/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java @@ -140,19 +140,15 @@ public class ExportServiceImpl implements ExportService { export.setState( Export.State.STARTED ); em.update( export ); - //retrieves all the organizations. Loops through them and backs up the appplications for each one - Map<UUID, String> organizationGet = getOrgs( config ); - for ( Map.Entry<UUID, String> organization : organizationGet.entrySet() ) { - try { - //exports all the applications for a single organization - exportApplicationsForOrg( organization, config, jobExecution ); - } - catch ( Exception e ) { - //if for any reason the backing up fails, then update the entity with a failed state. - export.setState( Export.State.FAILED ); - em.update( export ); - return; - } + try { + //exports all the applications for a single organization + exportApplicationForOrg( config.getOrganizationId(), config, jobExecution ); + } + catch ( Exception e ) { + //if for any reason the backing up fails, then update the entity with a failed state. + export.setState( Export.State.FAILED ); + em.update( export ); + return; } export.setState( Export.State.FINISHED ); em.update( export ); @@ -189,7 +185,6 @@ public class ExportServiceImpl implements ExportService { return organizationNames; } - public SchedulerService getSch() { return sch; } @@ -307,6 +302,79 @@ public class ExportServiceImpl implements ExportService { } } + //might be confusing, but uses the /s/ inclusion or exclusion nomenclature. + private void exportApplicationForOrg( UUID organizationUUID, final ExportInfo config, + final JobExecution jobExecution ) throws Exception { + + //retrieves export entity + UUID exportId = ( UUID ) jobExecution.getJobData().getProperty( EXPORT_ID ); + EntityManager exportManager = emf.getEntityManager( config.getApplicationId() ); + Export export = exportManager.get( exportId, Export.class ); + + //sets up a output stream for s3 backup. + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + + // Loop through the applications per organization + BiMap<UUID, String> applications = managementService.getApplicationsForOrganization( organizationUUID ); + for ( Map.Entry<UUID, String> application : applications.entrySet() ) { + + logger.info( application.getValue() + " : " + application.getKey() ); + + String appFileName = prepareOutputFileName( "application", application.getValue() ); + + JsonGenerator jg = getJsonGenerator( baos ); + + EntityManager em = emf.getEntityManager( application.getKey() ); + + jg.writeStartArray(); + + Map<String, Object> metadata = em.getApplicationCollectionMetadata(); + long starting_time = System.currentTimeMillis(); + + // Loop through the collections. This is the only way to loop + // through the entities in the application (former namespace). + //could support queries, just need to implement that in the rest endpoint. + for ( String collectionName : metadata.keySet() ) { + //if the collection you are looping through doesn't match the name of the one you want. Don't export it. + if ( collectionName.equals( config.getCollection() ) ) { + //Query entity manager for the entities in a collection + Query query = new Query(); + query.setLimit( MAX_ENTITY_FETCH ); + query.setResultsLevel( Results.Level.ALL_PROPERTIES ); + Results entities = em.searchCollection( em.getApplicationRef(), collectionName, query ); + + //pages through the query and backs up all results. + PagingResultsIterator itr = new PagingResultsIterator( entities ); + for ( Object e : itr ) { + starting_time = checkTimeDelta( starting_time, jobExecution ); + Entity entity = ( Entity ) e; + jg.writeStartObject(); + jg.writeFieldName( "Metadata" ); + jg.writeObject(entity ); + saveCollectionMembers( jg, em, config.getCollection(), entity ); + jg.writeEndObject(); + } + } + } + + // Close writer and file for this application. + jg.writeEndArray(); + jg.close(); + baos.flush(); + baos.close(); + + //sets up the Inputstream for copying the method to s3. + InputStream is = new ByteArrayInputStream( baos.toByteArray() ); + try { + s3Export.copyToS3( is, config, appFileName ); + } + catch ( Exception e ) { + export.setState( Export.State.FAILED ); + return; + } + } + } + /** * Regulates how long to wait until the next heartbeat. http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/152c72d4/stack/services/src/main/java/org/apache/usergrid/management/export/S3Export.java ---------------------------------------------------------------------- diff --git a/stack/services/src/main/java/org/apache/usergrid/management/export/S3Export.java b/stack/services/src/main/java/org/apache/usergrid/management/export/S3Export.java index 941a612..b774db8 100644 --- a/stack/services/src/main/java/org/apache/usergrid/management/export/S3Export.java +++ b/stack/services/src/main/java/org/apache/usergrid/management/export/S3Export.java @@ -13,4 +13,8 @@ import org.apache.usergrid.management.ExportInfo; public interface S3Export { void copyToS3( InputStream inputStream, ExportInfo exportInfo, String filename ); + String getFilename (); + + void setFilename (String givenName); + } http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/152c72d4/stack/services/src/main/java/org/apache/usergrid/management/export/S3ExportImpl.java ---------------------------------------------------------------------- diff --git a/stack/services/src/main/java/org/apache/usergrid/management/export/S3ExportImpl.java b/stack/services/src/main/java/org/apache/usergrid/management/export/S3ExportImpl.java index 2c274e3..1ca6e97 100644 --- a/stack/services/src/main/java/org/apache/usergrid/management/export/S3ExportImpl.java +++ b/stack/services/src/main/java/org/apache/usergrid/management/export/S3ExportImpl.java @@ -32,6 +32,7 @@ public class S3ExportImpl implements S3Export { @Override public void copyToS3( final InputStream inputStream, final ExportInfo exportInfo, String filename ) { + Logger logger = LoggerFactory.getLogger( ExportServiceImpl.class ); /*won't need any of the properties as I have the export info*/ String bucketName = exportInfo.getBucket_location(); @@ -80,4 +81,10 @@ public class S3ExportImpl implements S3Export { logger.error( "Error uploading to blob store", e ); } } + + @Override + public String getFilename () {return "";} + + @Override + public void setFilename(String givenName) {;} } http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/152c72d4/stack/services/src/test/java/org/apache/usergrid/management/cassandra/ManagementServiceIT.java ---------------------------------------------------------------------- diff --git a/stack/services/src/test/java/org/apache/usergrid/management/cassandra/ManagementServiceIT.java b/stack/services/src/test/java/org/apache/usergrid/management/cassandra/ManagementServiceIT.java index b4b82e0..70010d4 100644 --- a/stack/services/src/test/java/org/apache/usergrid/management/cassandra/ManagementServiceIT.java +++ b/stack/services/src/test/java/org/apache/usergrid/management/cassandra/ManagementServiceIT.java @@ -746,25 +746,29 @@ public class ManagementServiceIT { //Tests to make sure we can call the job with mock data and it runs. - @Ignore + @Ignore //Connections won't save when run with maven, but on local builds it will. public void testFileConnections() throws Exception { File f = null; try { - f = new File( "test.json" ); - f.delete(); + f = new File( "testFileConnections.json" ); + //f.delete(); } catch ( Exception e ) { //consumed because this checks to see if the file exists. If it doesn't then don't do anything and carry on. } + S3Export s3Export = new MockS3ExportImpl(); + s3Export.setFilename( "testFileConnections.json" ); + ExportService exportService = setup.getExportService(); HashMap<String, Object> payload = payloadBuilder(); ExportInfo exportInfo = new ExportInfo( payload ); + exportInfo.setOrganizationId( organization.getUuid() ); exportInfo.setApplicationId( applicationId ); exportInfo.setCollection( "users" ); @@ -823,7 +827,7 @@ public class ManagementServiceIT { File f = null; try { - f = new File( "test.json" ); + f = new File( "fileValidity.json" ); f.delete(); } catch ( Exception e ) { @@ -831,10 +835,12 @@ public class ManagementServiceIT { } S3Export s3Export = new MockS3ExportImpl(); + s3Export.setFilename( "fileValidity.json" ); ExportService exportService = setup.getExportService(); HashMap<String, Object> payload = payloadBuilder(); ExportInfo exportInfo = new ExportInfo( payload ); + exportInfo.setOrganizationId( organization.getUuid() ); exportInfo.setApplicationId( applicationId ); exportInfo.setCollection( "users" ); @@ -870,7 +876,7 @@ public class ManagementServiceIT { try { - f = new File( "test.json" ); + f = new File( "exportOneOrg.json" ); f.delete(); } catch ( Exception e ) { @@ -881,10 +887,12 @@ public class ManagementServiceIT { "123456789" ); S3Export s3Export = new MockS3ExportImpl(); + s3Export.setFilename("exportOneOrg.json"); ExportService exportService = setup.getExportService(); HashMap<String, Object> payload = payloadBuilder(); ExportInfo exportInfo = new ExportInfo( payload ); + exportInfo.setOrganizationId( organization.getUuid() ); exportInfo.setApplicationId( applicationId ); exportInfo.setCollection( "roles" ); @@ -925,7 +933,7 @@ public class ManagementServiceIT { String appName = "testAppNotExported"; try { - f = new File( "test.json" ); + f = new File( "exportOneApp.json" ); f.delete(); } catch ( Exception e ) { @@ -949,10 +957,12 @@ public class ManagementServiceIT { } S3Export s3Export = new MockS3ExportImpl(); + s3Export.setFilename( "exportOneApp.json" ); ExportService exportService = setup.getExportService(); HashMap<String, Object> payload = payloadBuilder(); ExportInfo exportInfo = new ExportInfo( payload ); + exportInfo.setOrganizationId( organization.getUuid() ); exportInfo.setApplicationId( applicationId ); exportInfo.setCollection( "roles" ); @@ -992,7 +1002,7 @@ public class ManagementServiceIT { // String appName = "ed-app"; try { - f = new File( "test.json" ); + f = new File( "exportOneCollection.json" ); f.delete(); } catch ( Exception e ) { @@ -1018,10 +1028,12 @@ public class ManagementServiceIT { } S3Export s3Export = new MockS3ExportImpl(); + s3Export.setFilename( "exportOneCollection.json" ); ExportService exportService = setup.getExportService(); HashMap<String, Object> payload = payloadBuilder(); ExportInfo exportInfo = new ExportInfo( payload ); + exportInfo.setOrganizationId( organization.getUuid() ); exportInfo.setApplicationId( applicationId ); exportInfo.setCollection( "baconators" ); @@ -1061,6 +1073,7 @@ public class ManagementServiceIT { HashMap<String, Object> payload = payloadBuilder(); ExportInfo exportInfo = new ExportInfo( payload ); + exportInfo.setOrganizationId( organization.getUuid() ); exportInfo.setApplicationId( applicationId ); @@ -1095,6 +1108,7 @@ public class ManagementServiceIT { JobExecution jobExecution = mock( JobExecution.class ); ExportInfo exportInfo = new ExportInfo( payload ); + exportInfo.setOrganizationId( organization.getUuid() ); exportInfo.setApplicationId( applicationId ); UUID entityExportUUID = eS.schedule( exportInfo ); http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/152c72d4/stack/services/src/test/java/org/apache/usergrid/management/cassandra/MockS3ExportImpl.java ---------------------------------------------------------------------- diff --git a/stack/services/src/test/java/org/apache/usergrid/management/cassandra/MockS3ExportImpl.java b/stack/services/src/test/java/org/apache/usergrid/management/cassandra/MockS3ExportImpl.java index d8a8b08..6419719 100644 --- a/stack/services/src/test/java/org/apache/usergrid/management/cassandra/MockS3ExportImpl.java +++ b/stack/services/src/test/java/org/apache/usergrid/management/cassandra/MockS3ExportImpl.java @@ -19,6 +19,7 @@ import org.apache.usergrid.management.export.S3Export; * Streams / reads the information written from the export service to a file named "test.json" */ public class MockS3ExportImpl implements S3Export { + public static String filename; @Override public void copyToS3( final InputStream inputStream, final ExportInfo exportInfo, String filename ) { Logger logger = LoggerFactory.getLogger( MockS3ExportImpl.class ); @@ -28,7 +29,7 @@ public class MockS3ExportImpl implements S3Export { //FileInputStream fis = new PrintWriter( inputStream ); try { - outputStream = new FileOutputStream( new File( "test.json" ) ); + outputStream = new FileOutputStream( new File( getFilename() ) ); } catch ( FileNotFoundException e ) { e.printStackTrace(); @@ -44,4 +45,12 @@ public class MockS3ExportImpl implements S3Export { e.printStackTrace(); } } + + @Override + public String getFilename () { + return filename; + } + + @Override + public void setFilename (String givenName) { filename = givenName; } }
