Updated endpoint such that the user must now specify ( through the endpoint ) what collection they want to backup. Updated tests to reflect the collection lookup Added collection name to export info.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/88a2a485 Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/88a2a485 Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/88a2a485 Branch: refs/heads/two-dot-o Commit: 88a2a485037f2c324c975fbcfb6cca5d5c30ae98 Parents: 1a2028d Author: grey <[email protected]> Authored: Thu Feb 27 15:41:06 2014 -0800 Committer: grey <[email protected]> Committed: Thu Feb 27 15:41:06 2014 -0800 ---------------------------------------------------------------------- .../applications/ApplicationResource.java | 9 +- .../rest/management/ManagementResourceIT.java | 8 +- .../apache/usergrid/management/ExportInfo.java | 6 +- .../management/export/ExportServiceImpl.java | 123 ++++++++++++------- .../cassandra/ManagementServiceIT.java | 77 +++++++++++- 5 files changed, 172 insertions(+), 51 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/88a2a485/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/applications/ApplicationResource.java ---------------------------------------------------------------------- diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/applications/ApplicationResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/applications/ApplicationResource.java index e94f4df..0e9d8ed 100644 --- a/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/applications/ApplicationResource.java +++ b/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/applications/ApplicationResource.java @@ -212,22 +212,24 @@ public class ApplicationResource extends AbstractContextResource { @POST - @Path("export") + @Path("collection/{collection_name}/export") @Consumes(APPLICATION_JSON) @RequireOrganizationAccess - public Response exportPostJson( @Context UriInfo ui, Map<String, Object> json, + public Response exportPostJson( @Context UriInfo ui,@PathParam( "collection_name" ) String collection_name ,Map<String, Object> json, @QueryParam("callback") @DefaultValue("") String callback ) throws OAuthSystemException { OAuthResponse response = null; UUID jobUUID = null; + String colExport = collection_name; Map<String, String> uuidRet = new HashMap<String, String>(); try { //parse the json into some useful object (the config params) ExportInfo objEx = new ExportInfo( json ); objEx.setApplicationId( applicationId ); + objEx.setCollection( colExport ); jobUUID = exportService.schedule( objEx ); uuidRet.put( "jobUUID", jobUUID.toString() ); } @@ -250,6 +252,7 @@ public class ApplicationResource extends AbstractContextResource { @GET + @RequireOrganizationAccess @Path("export/{jobUUID: [A-Fa-f0-9]{8}-[A-Fa-f0-9]{4}-[A-Fa-f0-9]{4}-[A-Fa-f0-9]{4}-[A-Fa-f0-9]{12}}") public Response exportGetJson( @Context UriInfo ui, @PathParam("jobUUID") UUID jobUUIDStr, @QueryParam("callback") @DefaultValue("") String callback ) throws Exception { @@ -269,4 +272,6 @@ public class ApplicationResource extends AbstractContextResource { return Response.status( SC_OK ).entity( entity.getState() ).build(); } + + } http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/88a2a485/stack/rest/src/test/java/org/apache/usergrid/rest/management/ManagementResourceIT.java ---------------------------------------------------------------------- diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/management/ManagementResourceIT.java b/stack/rest/src/test/java/org/apache/usergrid/rest/management/ManagementResourceIT.java index a36dfd3..a81d9fb 100644 --- a/stack/rest/src/test/java/org/apache/usergrid/rest/management/ManagementResourceIT.java +++ b/stack/rest/src/test/java/org/apache/usergrid/rest/management/ManagementResourceIT.java @@ -541,7 +541,7 @@ public class ManagementResourceIT extends AbstractRestIT { HashMap<String, Object> payload = payloadBuilder(); try { - node = resource().path( "/management/orgs/test-organization/apps/test-app/export" ) + node = resource().path( "/management/orgs/test-organization/apps/test-app/collection/users/export" ) .queryParam( "access_token", superAdminToken() ).accept( MediaType.APPLICATION_JSON ) .type( MediaType.APPLICATION_JSON_TYPE ).post( JsonNode.class, payload ); } @@ -606,7 +606,7 @@ public class ManagementResourceIT extends AbstractRestIT { HashMap<String, Object> payload = payloadBuilder(); try { - node = resource().path( "/management/orgs/test-organization/apps/test-app/export" ) + node = resource().path( "/management/orgs/test-organization/apps/test-app/collection/users/export" ) .queryParam( "access_token", superAdminToken() ).accept( MediaType.APPLICATION_JSON ) .type( MediaType.APPLICATION_JSON_TYPE ).post( JsonNode.class, payload ); } @@ -628,7 +628,7 @@ public class ManagementResourceIT extends AbstractRestIT { HashMap<String, Object> payload = payloadBuilder(); - node = resource().path( "/management/orgs/test-organization/apps/test-app/export" ) + node = resource().path( "/management/orgs/test-organization/apps/test-app/collection/users/export" ) .queryParam( "access_token", superAdminToken() ).accept( MediaType.APPLICATION_JSON ) .type( MediaType.APPLICATION_JSON_TYPE ).post( JsonNode.class, payload ); String uuid = String.valueOf( node.get( "jobUUID" ) ); @@ -673,7 +673,7 @@ public class ManagementResourceIT extends AbstractRestIT { Status responseStatus = Status.OK; UUID fake = UUID.fromString( "AAAAAAAA-FFFF-FFFF-FFFF-AAAAAAAAAAAA" ); try { - node = resource().path( "/management/orgs/test-organization/apps/test-app/export/" + fake ) + node = resource().path( "/management/orgs/test-organization/apps/test-app/collection/users/export/" + fake ) .accept( MediaType.APPLICATION_JSON ).type( MediaType.APPLICATION_JSON_TYPE ) .get( JsonNode.class ); } http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/88a2a485/stack/services/src/main/java/org/apache/usergrid/management/ExportInfo.java ---------------------------------------------------------------------- diff --git a/stack/services/src/main/java/org/apache/usergrid/management/ExportInfo.java b/stack/services/src/main/java/org/apache/usergrid/management/ExportInfo.java index 4436f0b..b1be504 100644 --- a/stack/services/src/main/java/org/apache/usergrid/management/ExportInfo.java +++ b/stack/services/src/main/java/org/apache/usergrid/management/ExportInfo.java @@ -20,7 +20,8 @@ public class ExportInfo extends TypedEntity { private String s3_accessId; private String s3_key; private String bucket_location; - private UUID applicationId; + private UUID applicationId; //TODO: include application when initializing export info + private String collection; //TODO: include collection when initializing export info public ExportInfo( Map<String, Object> exportData ) { @@ -33,6 +34,7 @@ public class ExportInfo extends TypedEntity { bucket_location = ( String ) storage_info.get( "bucket_location" ); } + public String getCollection() { return collection; } public UUID getApplicationId() { return applicationId; @@ -70,4 +72,6 @@ public class ExportInfo extends TypedEntity { public void setApplicationId( UUID appId ) { applicationId = appId;} + + public void setCollection ( String colName) { collection = colName; } } http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/88a2a485/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java ---------------------------------------------------------------------- diff --git a/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java b/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java index 37646a5..b716293 100644 --- a/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java +++ b/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java @@ -75,7 +75,7 @@ public class ExportServiceImpl implements ExportService { @Override public UUID schedule( final ExportInfo config ) throws Exception { - if(config == null){ + if ( config == null ) { logger.error( "export information cannot be null" ); return null; } @@ -84,18 +84,22 @@ public class ExportServiceImpl implements ExportService { Export export = new Export(); - //write to the em + //update state export = em.create( export ); export.setState( Export.State.CREATED ); em.update( export ); + //set data to be transferred to exportInfo JobData jobData = new JobData(); jobData.setProperty( "exportInfo", config ); jobData.setProperty( EXPORT_ID, export.getUuid() ); long soonestPossible = System.currentTimeMillis() + 250; //sch grace period + //schedule job sch.createJob( EXPORT_JOB_NAME, soonestPossible, jobData ); + + //update state export.setState( Export.State.SCHEDULED ); em.update( export ); @@ -104,14 +108,17 @@ public class ExportServiceImpl implements ExportService { /** - * Query Entity Manager for specific Export Entity within application - * + * Query Entity Manager for the string state of the Export Entity. + * This corresponds to the GET /export * @return String */ @Override public String getState( final UUID appId, final UUID uuid ) throws Exception { + //get application entity manager EntityManager rootEm = emf.getEntityManager( appId ); + + //retrieve the export entity. Export export = rootEm.get( uuid, Export.class ); if ( export == null ) { @@ -124,19 +131,26 @@ public class ExportServiceImpl implements ExportService { @Override public void doExport( final ExportInfo config, final JobExecution jobExecution ) throws Exception { + //get the entity manager for the application, and the entity that this Export corresponds to. UUID exportId = ( UUID ) jobExecution.getJobData().getProperty( EXPORT_ID ); EntityManager em = emf.getEntityManager( config.getApplicationId() ); Export export = em.get( exportId, Export.class ); + //update the entity state to show that the job has officially started. export.setState( Export.State.STARTED ); em.update( export ); + //retrieves all the organizations. Loops through them and backs up the appplications for each one Map<UUID, String> organizationGet = getOrgs( config ); for ( Map.Entry<UUID, String> organization : organizationGet.entrySet() ) { try { + //exports all the applications for a single organization exportApplicationsForOrg( organization, config, jobExecution ); - }catch(Exception e){ + } + catch ( Exception e ) { + //if for any reason the backing up fails, then update the entity with a failed state. export.setState( Export.State.FAILED ); + em.update( export ); return; } } @@ -145,6 +159,12 @@ public class ExportServiceImpl implements ExportService { } + /** + * Loops through all the organizations and returns a Map with the corresponding information + * @param exportInfo + * @return Map<UUID, String> + * @throws Exception + */ private Map<UUID, String> getOrgs( ExportInfo exportInfo ) throws Exception { // Loop through the organizations UUID orgId = null; @@ -204,13 +224,24 @@ public class ExportServiceImpl implements ExportService { //write test checking to see what happens if the input stream is closed or wrong. //TODO: make multipart streaming functional //currently only stores the collection in memory then flushes it. + + + /** + * Exports all applications for the given organization. + * @param organization + * @param config + * @param jobExecution + * @throws Exception + */ private void exportApplicationsForOrg( Map.Entry<UUID, String> organization, final ExportInfo config, final JobExecution jobExecution ) throws Exception { + //retrieves export entity UUID exportId = ( UUID ) jobExecution.getJobData().getProperty( EXPORT_ID ); EntityManager exportManager = emf.getEntityManager( config.getApplicationId() ); Export export = exportManager.get( exportId, Export.class ); + //sets up a output stream for s3 backup. ByteArrayOutputStream baos = new ByteArrayOutputStream(); logger.info( "" + organization ); @@ -236,26 +267,25 @@ public class ExportServiceImpl implements ExportService { // through the entities in the application (former namespace). //could support queries, just need to implement that in the rest endpoint. for ( String collectionName : metadata.keySet() ) { - if ( collectionName.equals( "exports" ) ) { - continue; - } - - Query query = new Query(); - query.setLimit( MAX_ENTITY_FETCH ); - query.setResultsLevel( Results.Level.ALL_PROPERTIES ); - Results entities = em.searchCollection( em.getApplicationRef(), collectionName, query ); - - PagingResultsIterator itr = new PagingResultsIterator( entities ); - - for ( Object e : itr ) { - starting_time = checkTimeDelta( starting_time, jobExecution ); - Entity entity = ( Entity ) e; - - jg.writeStartObject(); - jg.writeFieldName( "Metadata" ); - jg.writeObject( entity ); - saveCollectionMembers( jg, em, application.getValue(), entity ); - jg.writeEndObject(); + //if the collection you are looping through doesn't match the name of the one you want. Don't export it. + if ( collectionName.equals( config.getCollection() ) ) { + //Query entity manager for the entities in a collection + Query query = new Query(); + query.setLimit( MAX_ENTITY_FETCH ); + query.setResultsLevel( Results.Level.ALL_PROPERTIES ); + Results entities = em.searchCollection( em.getApplicationRef(), collectionName, query ); + + //pages through the query and backs up all results. + PagingResultsIterator itr = new PagingResultsIterator( entities ); + for ( Object e : itr ) { + starting_time = checkTimeDelta( starting_time, jobExecution ); + Entity entity = ( Entity ) e; + jg.writeStartObject(); + jg.writeFieldName( "Metadata" ); + jg.writeObject(entity ); + saveCollectionMembers( jg, em, config.getCollection(), entity ); + jg.writeEndObject(); + } } } @@ -265,10 +295,12 @@ public class ExportServiceImpl implements ExportService { baos.flush(); baos.close(); + //sets up the Inputstream for copying the method to s3. InputStream is = new ByteArrayInputStream( baos.toByteArray() ); try { s3Export.copyToS3( is, config, appFileName ); - }catch(Exception e){ + } + catch ( Exception e ) { export.setState( Export.State.FAILED ); return; } @@ -276,6 +308,12 @@ public class ExportServiceImpl implements ExportService { } + /** + * Regulates how long to wait until the next heartbeat. + * @param startingTime + * @param jobExecution + * @return + */ public long checkTimeDelta( long startingTime, final JobExecution jobExecution ) { long cur_time = System.currentTimeMillis(); @@ -292,38 +330,40 @@ public class ExportServiceImpl implements ExportService { * Serialize and save the collection members of this <code>entity</code> * * @param em Entity Manager - * @param application Application name + * @param collection Collection Name * @param entity entity */ - private void saveCollectionMembers( JsonGenerator jg, EntityManager em, String application, Entity entity ) + private void saveCollectionMembers( JsonGenerator jg, EntityManager em, String collection, Entity entity ) throws Exception { Set<String> collections = em.getCollections( entity ); - // Only create entry for Entities that have collections + // If your application doesn't have any e if ( ( collections == null ) || collections.isEmpty() ) { return; } for ( String collectionName : collections ) { - jg.writeFieldName( collectionName ); - jg.writeStartArray(); + if ( collectionName.equals( collection ) ) { + jg.writeFieldName( collectionName ); + jg.writeStartArray(); - //is 100000 an arbitary number? - Results collectionMembers = - em.getCollection( entity, collectionName, null, 100000, Results.Level.IDS, false ); + //is 100000 an arbitary number? + Results collectionMembers = + em.getCollection( entity, collectionName, null, 100000, Results.Level.IDS, false ); - List<UUID> entityIds = collectionMembers.getIds(); + List<UUID> entityIds = collectionMembers.getIds(); - if ( ( entityIds != null ) && !entityIds.isEmpty() ) { - for ( UUID childEntityUUID : entityIds ) { - jg.writeObject( childEntityUUID.toString() ); + if ( ( entityIds != null ) && !entityIds.isEmpty() ) { + for ( UUID childEntityUUID : entityIds ) { + jg.writeObject( childEntityUUID.toString() ); + } } - } - // End collection array. - jg.writeEndArray(); + // End collection array. + jg.writeEndArray(); + } } // Write connections @@ -331,7 +371,6 @@ public class ExportServiceImpl implements ExportService { // Write dictionaries saveDictionaries( entity, em, jg ); - } http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/88a2a485/stack/services/src/test/java/org/apache/usergrid/management/cassandra/ManagementServiceIT.java ---------------------------------------------------------------------- diff --git a/stack/services/src/test/java/org/apache/usergrid/management/cassandra/ManagementServiceIT.java b/stack/services/src/test/java/org/apache/usergrid/management/cassandra/ManagementServiceIT.java index 0b1f7be..b4b82e0 100644 --- a/stack/services/src/test/java/org/apache/usergrid/management/cassandra/ManagementServiceIT.java +++ b/stack/services/src/test/java/org/apache/usergrid/management/cassandra/ManagementServiceIT.java @@ -766,6 +766,7 @@ public class ManagementServiceIT { ExportInfo exportInfo = new ExportInfo( payload ); exportInfo.setApplicationId( applicationId ); + exportInfo.setCollection( "users" ); EntityManager em = setup.getEmf().getEntityManager( applicationId ); //intialize user object to be posted @@ -835,6 +836,7 @@ public class ManagementServiceIT { ExportInfo exportInfo = new ExportInfo( payload ); exportInfo.setApplicationId( applicationId ); + exportInfo.setCollection( "users" ); UUID exportUUID = exportService.schedule( exportInfo ); exportService.setS3Export( s3Export ); @@ -884,6 +886,7 @@ public class ManagementServiceIT { ExportInfo exportInfo = new ExportInfo( payload ); exportInfo.setApplicationId( applicationId ); + exportInfo.setCollection( "roles" ); UUID exportUUID = exportService.schedule( exportInfo ); exportService.setS3Export( s3Export ); @@ -902,7 +905,7 @@ public class ManagementServiceIT { org.json.simple.JSONArray a = ( org.json.simple.JSONArray ) parser.parse( new FileReader( f ) ); - //assertEquals( 3 , a.size() ); + assertEquals( 3 , a.size() ); for ( int i = 0; i < a.size(); i++ ) { org.json.simple.JSONObject entity = ( org.json.simple.JSONObject ) a.get( i ); org.json.simple.JSONObject entityData = ( JSONObject ) entity.get( "Metadata" ); @@ -951,6 +954,7 @@ public class ManagementServiceIT { ExportInfo exportInfo = new ExportInfo( payload ); exportInfo.setApplicationId( applicationId ); + exportInfo.setCollection( "roles" ); UUID exportUUID = exportService.schedule( exportInfo ); exportService.setS3Export( s3Export ); @@ -969,7 +973,76 @@ public class ManagementServiceIT { org.json.simple.JSONArray a = ( org.json.simple.JSONArray ) parser.parse( new FileReader( f ) ); - //assertEquals( 3 , a.size() ); + assertEquals( 3 , a.size() ); + for ( int i = 0; i < a.size(); i++ ) { + org.json.simple.JSONObject data = ( org.json.simple.JSONObject ) a.get( i ); + org.json.simple.JSONObject entityData = ( JSONObject ) data.get( "Metadata" ); + String entityName = ( String ) entityData.get( "name" ); + assertFalse( "junkRealName".equals( entityName ) ); + //assertNotEquals( "NotEquals","junkRealName",entityName ); + } + f.delete(); + } + + @Test + public void testFileExportOneCollection() throws Exception { + + File f = null; +// String orgName = "ed-organization"; +// String appName = "ed-app"; + + try { + f = new File( "test.json" ); + f.delete(); + } + catch ( Exception e ) { + //consumed because this checks to see if the file exists. If it doesn't, don't do anything and carry on. + } + + //UUID appId = setup.getEmf().createApplication( orgName, appName ); + + + + EntityManager em = setup.getEmf().getEntityManager( applicationId); + em.createApplicationCollection( "baconators" ); + //intialize user object to be posted + Map<String, Object> userProperties = null; + Entity[] entity; + entity = new Entity[10]; + //creates entities + for ( int i = 0; i < 10; i++ ) { + userProperties = new LinkedHashMap<String, Object>(); + userProperties.put( "username", "billybob" + i ); + userProperties.put( "email", "test" + i + "@anuff.com" );//String.format( "[email protected]", i ) ); + entity[i] = em.create( "baconators", userProperties ); + } + + S3Export s3Export = new MockS3ExportImpl(); + ExportService exportService = setup.getExportService(); + HashMap<String, Object> payload = payloadBuilder(); + + ExportInfo exportInfo = new ExportInfo( payload ); + exportInfo.setApplicationId( applicationId ); + exportInfo.setCollection( "baconators" ); + + UUID exportUUID = exportService.schedule( exportInfo ); + exportService.setS3Export( s3Export ); + + JobData jobData = new JobData(); + jobData.setProperty( "jobName", "exportJob" ); + jobData.setProperty( "exportInfo", exportInfo ); + jobData.setProperty( "exportId", exportUUID ); + + JobExecution jobExecution = mock( JobExecution.class ); + when( jobExecution.getJobData() ).thenReturn( jobData ); + + exportService.doExport( exportInfo, jobExecution ); + + JSONParser parser = new JSONParser(); + + org.json.simple.JSONArray a = ( org.json.simple.JSONArray ) parser.parse( new FileReader( f ) ); + + assertEquals( 10 , a.size() ); for ( int i = 0; i < a.size(); i++ ) { org.json.simple.JSONObject data = ( org.json.simple.JSONObject ) a.get( i ); org.json.simple.JSONObject entityData = ( JSONObject ) data.get( "Metadata" );
