Added different states. New States that are in use are currently: CREATED,FAILED,SCHEDULED,STARTED,FINISHED. Support for canceled and expired at a later date.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/41caff77 Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/41caff77 Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/41caff77 Branch: refs/pull/70/merge Commit: 41caff773cb5165ff21252903a66710970564a5c Parents: 96a1a72 Author: grey <[email protected]> Authored: Tue Feb 25 14:01:40 2014 -0800 Committer: grey <[email protected]> Committed: Tue Feb 25 14:01:40 2014 -0800 ---------------------------------------------------------------------- .../usergrid/persistence/entities/Export.java | 5 +- .../rest/management/ManagementResourceIT.java | 2 +- .../usergrid/management/export/ExportJob.java | 33 +++++++-- .../management/export/ExportServiceImpl.java | 77 +++++++------------- .../cassandra/ManagementServiceIT.java | 32 ++++---- 5 files changed, 70 insertions(+), 79 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/41caff77/stack/core/src/main/java/org/apache/usergrid/persistence/entities/Export.java ---------------------------------------------------------------------- diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/entities/Export.java b/stack/core/src/main/java/org/apache/usergrid/persistence/entities/Export.java index 7905550..1719412 100644 --- a/stack/core/src/main/java/org/apache/usergrid/persistence/entities/Export.java +++ b/stack/core/src/main/java/org/apache/usergrid/persistence/entities/Export.java @@ -14,10 +14,9 @@ import org.apache.usergrid.persistence.annotations.EntityProperty; */ @XmlRootElement public class Export extends TypedEntity { - + //canceled , and expired states aren't used in current iteration. public static enum State { - //CREATED, FAILED, SCHEDULED, STARTED, FINISHED, CANCELED, EXPIRED - PENDING, STARTED, FAILED, COMPLETED + CREATED, FAILED, SCHEDULED, STARTED, FINISHED, CANCELED, EXPIRED } http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/41caff77/stack/rest/src/test/java/org/apache/usergrid/rest/management/ManagementResourceIT.java ---------------------------------------------------------------------- diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/management/ManagementResourceIT.java b/stack/rest/src/test/java/org/apache/usergrid/rest/management/ManagementResourceIT.java index a9b06fd..a36dfd3 100644 --- a/stack/rest/src/test/java/org/apache/usergrid/rest/management/ManagementResourceIT.java +++ b/stack/rest/src/test/java/org/apache/usergrid/rest/management/ManagementResourceIT.java @@ -645,7 +645,7 @@ public class ManagementResourceIT extends AbstractRestIT { assertEquals( Status.OK, responseStatus ); - assertEquals( "PENDING", node.asText() );//TODO: do tests for other states in service tier + assertEquals( "SCHEDULED", node.asText() );//TODO: do tests for other states in service tier } http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/41caff77/stack/services/src/main/java/org/apache/usergrid/management/export/ExportJob.java ---------------------------------------------------------------------- diff --git a/stack/services/src/main/java/org/apache/usergrid/management/export/ExportJob.java b/stack/services/src/main/java/org/apache/usergrid/management/export/ExportJob.java index ed2fef5..83cda0a 100644 --- a/stack/services/src/main/java/org/apache/usergrid/management/export/ExportJob.java +++ b/stack/services/src/main/java/org/apache/usergrid/management/export/ExportJob.java @@ -1,6 +1,8 @@ package org.apache.usergrid.management.export; +import java.util.UUID; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -9,6 +11,9 @@ import org.springframework.stereotype.Component; import org.apache.usergrid.batch.JobExecution; import org.apache.usergrid.batch.job.OnlyOnceJob; import org.apache.usergrid.management.ExportInfo; +import org.apache.usergrid.persistence.EntityManager; +import org.apache.usergrid.persistence.EntityManagerFactory; +import org.apache.usergrid.persistence.entities.Export; import org.apache.usergrid.persistence.entities.JobData; @@ -18,36 +23,52 @@ import org.apache.usergrid.persistence.entities.JobData; */ @Component("exportJob") public class ExportJob extends OnlyOnceJob { + public static final String EXPORT_ID = "exportId"; private static final Logger logger = LoggerFactory.getLogger( ExportJob.class ); @Autowired ExportService exportService; + @Autowired + private EntityManagerFactory emf; + public ExportJob() { logger.info( "ExportJob created " + this ); } - +//need to add in checking the export state and what happens if it fails. This should then update the export entity with failed state. @Override public void doJob( JobExecution jobExecution ) throws Exception { logger.info( "execute ExportJob {}", jobExecution ); + UUID exportId = ( UUID ) jobExecution.getJobData().getProperty( EXPORT_ID ); + //as long as I have the entity UUID I should be able to find it from anywhere right? + + ExportInfo config = null; JobData jobData = jobExecution.getJobData(); - config = ( ExportInfo ) jobData.getProperty( "exportInfo" ); if ( jobData == null ) { logger.error( "jobData cannot be null" ); return; } - else if ( config == null ) { + config = ( ExportInfo ) jobData.getProperty( "exportInfo" ); + if ( config == null ) { logger.error( "Export information cannot be null" ); return; } - else { + EntityManager em = emf.getEntityManager( config.getApplicationId() ); + Export export = em.get( exportId, Export.class ); + jobExecution.heartbeat(); - exportService.doExport( config, jobExecution ); - } + try { + exportService.doExport( config, jobExecution ); + }catch (Exception e) { + logger.error( "Export Service failed to complete job" ); + export.setState( Export.State.FAILED ); + em.update( export ); + } + logger.info( "executed ExportJob process completed" ); } http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/41caff77/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java ---------------------------------------------------------------------- diff --git a/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java b/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java index 155639c..37646a5 100644 --- a/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java +++ b/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java @@ -75,39 +75,18 @@ public class ExportServiceImpl implements ExportService { @Override public UUID schedule( final ExportInfo config ) throws Exception { - EntityManager em = emf.getEntityManager( config.getApplicationId() ); - - Export export = new Export(); - export.setState( Export.State.PENDING ); - - //validate that org exists,then app, then collection. - - String pathToBeParsed = config.getPath(); - //split the path so that you can verify that the organization and the app exist. - String[] pathItems = pathToBeParsed.split( "/" ); - - - try { - managementService.getOrganizationByName( pathItems[0] ); - } - catch ( Exception e ) { - logger.error( "Organization doesn't exist" ); - } - - try { - managementService.getApplicationInfo( pathItems[1] ); - } - catch ( Exception e ) { - logger.error( "Application doesn't exist" ); + if(config == null){ + logger.error( "export information cannot be null" ); + return null; } + EntityManager em = emf.getEntityManager( config.getApplicationId() ); - //TODO: parse path and make sure all the things you need actually exist. then throw - // good error messages when not found. + Export export = new Export(); //write to the em export = em.create( export ); - export.setState( Export.State.PENDING ); + export.setState( Export.State.CREATED ); em.update( export ); JobData jobData = new JobData(); @@ -117,6 +96,8 @@ public class ExportServiceImpl implements ExportService { long soonestPossible = System.currentTimeMillis() + 250; //sch grace period sch.createJob( EXPORT_JOB_NAME, soonestPossible, jobData ); + export.setState( Export.State.SCHEDULED ); + em.update( export ); return export.getUuid(); } @@ -147,34 +128,25 @@ public class ExportServiceImpl implements ExportService { EntityManager em = emf.getEntityManager( config.getApplicationId() ); Export export = em.get( exportId, Export.class ); - String pathToBeParsed = config.getPath(); - String[] pathItems = pathToBeParsed.split( "/" ); - try { - managementService.getOrganizationByName( pathItems[0] ); - } - catch ( Exception e ) { - logger.error( "Organization doesn't exist" ); - return; - } - - //update state and re-write the entity export.setState( Export.State.STARTED ); - em.update( export ); Map<UUID, String> organizationGet = getOrgs( config ); for ( Map.Entry<UUID, String> organization : organizationGet.entrySet() ) { - //needs to pass app name, and possibly collection to export - exportApplicationsForOrg( organization, config, jobExecution ); + try { + exportApplicationsForOrg( organization, config, jobExecution ); + }catch(Exception e){ + export.setState( Export.State.FAILED ); + return; + } } - export.setState( Export.State.COMPLETED ); + export.setState( Export.State.FINISHED ); em.update( export ); } private Map<UUID, String> getOrgs( ExportInfo exportInfo ) throws Exception { // Loop through the organizations - // TODO:this will come from the orgs in schedule when you do the validations. delete orgId UUID orgId = null; Map<UUID, String> organizationNames = null; @@ -183,7 +155,7 @@ public class ExportServiceImpl implements ExportService { organizationNames = managementService.getOrganizations(); } - else { + else {//this case isn't used yet, but might be in the future. OrganizationInfo info = managementService.getOrganizationByUuid( orgId ); if ( info == null ) { @@ -194,7 +166,6 @@ public class ExportServiceImpl implements ExportService { organizationNames.put( orgId, info.getName() ); } - return organizationNames; } @@ -236,6 +207,10 @@ public class ExportServiceImpl implements ExportService { private void exportApplicationsForOrg( Map.Entry<UUID, String> organization, final ExportInfo config, final JobExecution jobExecution ) throws Exception { + UUID exportId = ( UUID ) jobExecution.getJobData().getProperty( EXPORT_ID ); + EntityManager exportManager = emf.getEntityManager( config.getApplicationId() ); + Export export = exportManager.get( exportId, Export.class ); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); logger.info( "" + organization ); @@ -291,7 +266,12 @@ public class ExportServiceImpl implements ExportService { baos.close(); InputStream is = new ByteArrayInputStream( baos.toByteArray() ); - s3Export.copyToS3( is, config, appFileName ); + try { + s3Export.copyToS3( is, config, appFileName ); + }catch(Exception e){ + export.setState( Export.State.FAILED ); + return; + } } } @@ -319,14 +299,12 @@ public class ExportServiceImpl implements ExportService { throws Exception { Set<String> collections = em.getCollections( entity ); - //jg.writeStartObject(); // Only create entry for Entities that have collections if ( ( collections == null ) || collections.isEmpty() ) { return; } - for ( String collectionName : collections ) { jg.writeFieldName( collectionName ); @@ -354,8 +332,6 @@ public class ExportServiceImpl implements ExportService { // Write dictionaries saveDictionaries( entity, em, jg ); - // End the object if it was Started - //jg.writeEndObject(); } @@ -448,7 +424,6 @@ public class ExportServiceImpl implements ExportService { str.append( ".json" ); String outputFileName = str.toString(); - //TODO:this is , i feel, bad practice so make sure to come back here and fix it up. return outputFileName; } http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/41caff77/stack/services/src/test/java/org/apache/usergrid/management/cassandra/ManagementServiceIT.java ---------------------------------------------------------------------- diff --git a/stack/services/src/test/java/org/apache/usergrid/management/cassandra/ManagementServiceIT.java b/stack/services/src/test/java/org/apache/usergrid/management/cassandra/ManagementServiceIT.java index 34affe6..0b1f7be 100644 --- a/stack/services/src/test/java/org/apache/usergrid/management/cassandra/ManagementServiceIT.java +++ b/stack/services/src/test/java/org/apache/usergrid/management/cassandra/ManagementServiceIT.java @@ -18,13 +18,6 @@ import org.junit.Rule; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.apache.usergrid.management.ExportInfo; -import org.apache.usergrid.management.export.ExportJob; -import org.apache.usergrid.management.export.ExportService; -import org.apache.usergrid.management.export.S3Export; -import org.apache.usergrid.management.export.S3ExportImpl; -import org.apache.usergrid.persistence.EntityManagerFactory; -import org.apache.usergrid.persistence.entities.Export; import org.apache.usergrid.ServiceITSetup; import org.apache.usergrid.ServiceITSetupImpl; @@ -33,11 +26,19 @@ import org.apache.usergrid.batch.JobExecution; import org.apache.usergrid.cassandra.CassandraResource; import org.apache.usergrid.cassandra.ClearShiroSubject; import org.apache.usergrid.cassandra.Concurrent; +import org.apache.usergrid.count.SimpleBatcher; +import org.apache.usergrid.management.ExportInfo; import org.apache.usergrid.management.OrganizationInfo; import org.apache.usergrid.management.UserInfo; +import org.apache.usergrid.management.export.ExportJob; +import org.apache.usergrid.management.export.ExportService; +import org.apache.usergrid.management.export.S3Export; +import org.apache.usergrid.management.export.S3ExportImpl; import org.apache.usergrid.persistence.CredentialsInfo; import org.apache.usergrid.persistence.Entity; import org.apache.usergrid.persistence.EntityManager; +import org.apache.usergrid.persistence.EntityManagerFactory; +import org.apache.usergrid.persistence.entities.Export; import org.apache.usergrid.persistence.entities.JobData; import org.apache.usergrid.persistence.entities.User; import org.apache.usergrid.security.AuthPrincipalType; @@ -48,16 +49,14 @@ import org.apache.usergrid.security.tokens.exceptions.InvalidTokenException; import org.apache.usergrid.utils.JsonUtils; import org.apache.usergrid.utils.UUIDUtils; -import org.apache.usergrid.count.SimpleBatcher; - import static org.apache.commons.codec.binary.Base64.encodeBase64URLSafeString; +import static org.apache.usergrid.persistence.Schema.DICTIONARY_CREDENTIALS; +import static org.apache.usergrid.persistence.cassandra.CassandraService.MANAGEMENT_APPLICATION_ID; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; -import static org.apache.usergrid.persistence.Schema.DICTIONARY_CREDENTIALS; -import static org.apache.usergrid.persistence.cassandra.CassandraService.MANAGEMENT_APPLICATION_ID; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -862,7 +861,6 @@ public class ManagementServiceIT { f.delete(); } - @Test public void testFileExportOneOrg() throws Exception { @@ -1013,7 +1011,6 @@ public class ManagementServiceIT { assert ( true ); } - @Test public void testExportDoExport() throws Exception { @@ -1041,20 +1038,19 @@ public class ManagementServiceIT { Export exportEntity = ( Export ) em.get( entityExportUUID ); assertNotNull( exportEntity ); String derp = exportEntity.getState().name(); - assertEquals( "PENDING", exportEntity.getState().name() ); + assertEquals( "SCHEDULED", exportEntity.getState().name() ); try { eS.doExport( exportInfo, jobExecution ); } catch ( Exception e ) { - throw e; - //assert(false); + //throw e; + assert(false); } exportEntity = ( Export ) em.get( entityExportUUID ); assertNotNull( exportEntity ); - assertEquals( "COMPLETED", exportEntity.getState().name() ); + assertEquals( "FINISHED", exportEntity.getState().name() ); } - //tests that with empty job data, the export still runs. @Test public void testExportEmptyJobData() throws Exception {
