Added a batch job that will eventually need to be converted to a only one job instance. First pass doesnât need multiple jobs Will also need to have the job launch the schedule and have it be a component then have THAT method call the doExport method. Added some logging to let you debug if there is no organization or no application. dummy data is set for the UUID as there is no class to hold that information yet. Added a comment to export info that dictates the next steps to get jobs working. Added a new test that creates 100 entities, but the test doesnât contain file checking yet. Re included authorization, but will most likely be taken out. Also re added the sch command but it will to call the doExport not just call both methods.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/b45a6fdc Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/b45a6fdc Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/b45a6fdc Branch: refs/pull/70/merge Commit: b45a6fdce32d9fe54ef6702a59981ad3360d9cb6 Parents: 7a8ec96 Author: GERey <[email protected]> Authored: Fri Feb 7 20:38:00 2014 -0800 Committer: GERey <[email protected]> Committed: Fri Feb 7 20:38:00 2014 -0800 ---------------------------------------------------------------------- .../rest/management/ManagementResource.java | 9 ++-- .../rest/management/ManagementResourceIT.java | 46 +++++++++++++++++++- .../org/usergrid/management/ExportInfo.java | 20 ++++++++- .../management/export/ExportBatchJob.java | 36 +++++++++++++++ .../management/export/ExportServiceImpl.java | 40 +++++++++++++++-- 5 files changed, 141 insertions(+), 10 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b45a6fdc/stack/rest/src/main/java/org/usergrid/rest/management/ManagementResource.java ---------------------------------------------------------------------- diff --git a/stack/rest/src/main/java/org/usergrid/rest/management/ManagementResource.java b/stack/rest/src/main/java/org/usergrid/rest/management/ManagementResource.java index b2e6d36..4985239 100644 --- a/stack/rest/src/main/java/org/usergrid/rest/management/ManagementResource.java +++ b/stack/rest/src/main/java/org/usergrid/rest/management/ManagementResource.java @@ -450,18 +450,21 @@ public class ManagementResource extends AbstractContextResource { @POST @Path( "export" ) @Consumes(MediaType.APPLICATION_JSON) - public Response exportPostJson (@Context UriInfo ui, Map<String, Object> json, + public Response exportPostJson (@Context UriInfo ui,@HeaderParam( "Authorization" ) String authorization, + Map<String, Object> json, @QueryParam( "callback" ) @DefaultValue( "" ) String callback){ OAuthResponse response = null; + String client_id; + String client_secret; + try { //parse the json into some useful object (the config params) ExportInfo objEx = new ExportInfo(json); - - //exportService.schedule(objEx); + exportService.schedule(objEx); exportService.doExport( objEx ); } catch (Exception e) { http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b45a6fdc/stack/rest/src/test/java/org/usergrid/rest/management/ManagementResourceIT.java ---------------------------------------------------------------------- diff --git a/stack/rest/src/test/java/org/usergrid/rest/management/ManagementResourceIT.java b/stack/rest/src/test/java/org/usergrid/rest/management/ManagementResourceIT.java index 0d6f8e0..6e54feb 100644 --- a/stack/rest/src/test/java/org/usergrid/rest/management/ManagementResourceIT.java +++ b/stack/rest/src/test/java/org/usergrid/rest/management/ManagementResourceIT.java @@ -22,6 +22,7 @@ import java.util.Map; import javax.ws.rs.core.MediaType; import org.codehaus.jackson.JsonNode; +import org.junit.Ignore; import org.junit.Test; import org.usergrid.cassandra.Concurrent; import org.usergrid.management.OrganizationInfo; @@ -527,8 +528,9 @@ public class ManagementResourceIT extends AbstractRestIT { //assertEquals( Status.OK, status ); } - - @Test +//test needs to be updated to do validation of files/ and validation of file created in s3. + // eventually will need to simulate dropped connections as well. + @Ignore public void exportCallSuccessful() throws Exception { Status responseStatus = Status.OK; JsonNode node = null; @@ -559,4 +561,44 @@ public class ManagementResourceIT extends AbstractRestIT { assertEquals( Status.OK, responseStatus ); } + + @Ignore + public void exportCallCreationEntities100() throws Exception { + Status responseStatus = Status.OK; + JsonNode node = null; + + HashMap<String, Object> payload = new HashMap<String, Object>(); + Map<String, Object> properties = new HashMap<String, Object>(); + Map<String, Object> storage_info = new HashMap<String, Object>(); + //TODO: make sure to put a valid admin token here. + storage_info.put( "admin_token", adminToken() ); + //TODO: always put dummy values here and ignore this test. + storage_info.put( "s3_key","insert key here" ); + storage_info.put( "s3_accessId","insert access id here"); + storage_info.put( "bucket_location","insert bucket name here"); + + properties.put( "storage_provider","s3"); + properties.put( "storage_info",storage_info); + + payload.put( "path", "test-organization/test-app/user"); + payload.put( "properties", properties); + + for (int i = 0; i < 100; i++) { + Map<String, String> userCreation = hashMap( "type", "app_user" ).map( "name", "fred"+i ); + + node = resource().path( "/test-organization/test-app/app_users" ) + .queryParam( "access_token", access_token ).accept( MediaType.APPLICATION_JSON ) + .type( MediaType.APPLICATION_JSON_TYPE ).post( JsonNode.class, userCreation ); + } + + try { + node = resource().path( "/management/export" ).accept( MediaType.APPLICATION_JSON ) + .type( MediaType.APPLICATION_JSON_TYPE ).post( JsonNode.class, payload ); + } + catch ( UniformInterfaceException uie ) { + responseStatus = uie.getResponse().getClientResponseStatus(); + } + + assertEquals( Status.OK, responseStatus ); + } } http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b45a6fdc/stack/services/src/main/java/org/usergrid/management/ExportInfo.java ---------------------------------------------------------------------- diff --git a/stack/services/src/main/java/org/usergrid/management/ExportInfo.java b/stack/services/src/main/java/org/usergrid/management/ExportInfo.java index 5dbb771..1a272e4 100644 --- a/stack/services/src/main/java/org/usergrid/management/ExportInfo.java +++ b/stack/services/src/main/java/org/usergrid/management/ExportInfo.java @@ -6,7 +6,7 @@ import java.util.Map; * Created by ApigeeCorporation on 1/31/14. */ //TODO: Documentation on this class. -public class ExportInfo { +public class ExportInfo { //extends Entity something { private String path; private Map<String, Object> properties; @@ -17,6 +17,24 @@ public class ExportInfo { private String s3_key; private String bucket_location; + + /** + * + *the sch system doesn't make any assumes about the job or how it works. + * and so if I need additional information to be persistant. + * + * The way to save data between queue and storing it. + * + * in my case, create a export entity. before I schedule the job and it'll have the pending state in it and + * all the information I need to run. Then I'll pass the ID of the export info I saved in a collection and i'll put that in the jbo + * data. + * + * persist the state in mechanisum that they can all access. + * + * I could make it a class and I can make it an entity. That way I can get it in and out. + * doesn't get exposed to the user. + */ + public ExportInfo ( Map<String, Object> exportData) { path = (String) exportData.get("path"); properties = (Map) exportData.get("properties"); http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b45a6fdc/stack/services/src/main/java/org/usergrid/management/export/ExportBatchJob.java ---------------------------------------------------------------------- diff --git a/stack/services/src/main/java/org/usergrid/management/export/ExportBatchJob.java b/stack/services/src/main/java/org/usergrid/management/export/ExportBatchJob.java new file mode 100644 index 0000000..08f3701 --- /dev/null +++ b/stack/services/src/main/java/org/usergrid/management/export/ExportBatchJob.java @@ -0,0 +1,36 @@ +package org.usergrid.management.export; + +import java.util.UUID; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.usergrid.batch.Job; +import org.usergrid.batch.JobExecution; +import org.usergrid.persistence.entities.JobData; + + +/** + * + * + */ +public class ExportBatchJob implements Job { + + private static final Logger logger = LoggerFactory.getLogger( ExportBatchJob.class ); + + //TODO: ask if the service manager is needed here. Maybe in order to see if process. + @Autowired + protected ExportService exportService; + + + @Override + public void execute( final JobExecution execution ) throws Exception { + + logger.info("execute ExportBatchJob {}", execution); + + JobData jobData = execution.getJobData(); + UUID jobUUID = (UUID) jobData.getProperty( "jobUUID" ); + + + } +} http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b45a6fdc/stack/services/src/main/java/org/usergrid/management/export/ExportServiceImpl.java ---------------------------------------------------------------------- diff --git a/stack/services/src/main/java/org/usergrid/management/export/ExportServiceImpl.java b/stack/services/src/main/java/org/usergrid/management/export/ExportServiceImpl.java index 5fd6a58..d34fb23 100644 --- a/stack/services/src/main/java/org/usergrid/management/export/ExportServiceImpl.java +++ b/stack/services/src/main/java/org/usergrid/management/export/ExportServiceImpl.java @@ -36,6 +36,7 @@ import org.usergrid.persistence.EntityManagerFactory; import org.usergrid.persistence.Query; import org.usergrid.persistence.Results; import org.usergrid.persistence.cassandra.CassandraService; +import org.usergrid.persistence.entities.JobData; import com.google.common.collect.BiMap; import com.google.common.collect.ImmutableSet; @@ -49,6 +50,8 @@ import com.google.inject.Module; */ public class ExportServiceImpl implements ExportService{ + + private static final Logger logger = LoggerFactory.getLogger( ExportServiceImpl.class ); //dependency injection //inject scheduler - needs to be auto wired private SchedulerService sch; @@ -80,11 +83,39 @@ public class ExportServiceImpl implements ExportService{ @Override public void schedule( final ExportInfo config ) { - //SchedulerServiceImpl sch = - //validate org exists - //validate user has access to or + //validate that org exists,then app, then collection. + String pathToBeParsed = config.getPath(); + //split the path so that you can verify that the organization and the app exist. + String[] pathItems = pathToBeParsed.split( "/" ); + try { + managementService.getOrganizationByName( pathItems[0] ); + } + catch ( Exception e ) { + logger.error( "Organization doesn't exist" ); + } + + try { + managementService.getApplicationInfo( pathItems[1] ); + } + catch ( Exception e ) { + logger.error( "Application doesn't exist" ); + } + + + //TODO: parse path and make sure all the things you need actually exist. then throw + // good error messages when not found. + + // managementService.getOrganizationByName( ) + //validate user has access key to org (rather valid user has admin access token) + //this is token validation //schedule the job + JobData jobData = new JobData(); + + jobData.setProperty( "jobId", "0001" );//TODO: store uuid here, give export job uuid. + long soonestPossible = System.currentTimeMillis() + 250; //sch grace period + sch.createJob( "queueExportJob",soonestPossible, jobData ); + } @@ -123,6 +154,7 @@ public class ExportServiceImpl implements ExportService{ if ( info == null ) { //logger.error( "Organization info is null!" ); + //TODO: remove all instances of system.exit in code case that was adapated. System.exit( 1 ); } @@ -168,7 +200,7 @@ public class ExportServiceImpl implements ExportService{ private void exportApplicationsForOrg( Map.Entry<UUID, String> organization,final ExportInfo config ) throws Exception { - Logger logger = LoggerFactory.getLogger( ExportServiceImpl.class ); + logger.info( "" + organization );
