http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/a7840164/stack/rest/src/test/java/org/apache/usergrid/rest/management/ManagementResourceIT.java ---------------------------------------------------------------------- diff --cc stack/rest/src/test/java/org/apache/usergrid/rest/management/ManagementResourceIT.java index 9b6e842,cec172d..7955ebc --- a/stack/rest/src/test/java/org/apache/usergrid/rest/management/ManagementResourceIT.java +++ b/stack/rest/src/test/java/org/apache/usergrid/rest/management/ManagementResourceIT.java @@@ -17,86 -17,176 +17,103 @@@ package org.apache.usergrid.rest.management; --import java.util.ArrayList; --import java.util.HashMap; --import java.util.LinkedHashMap; --import java.util.List; --import java.util.Map; --import java.util.UUID; -- --import javax.ws.rs.core.MediaType; -- +import com.fasterxml.jackson.databind.JsonNode; - - import org.junit.Rule; - import org.junit.Test; - - import org.apache.commons.lang.StringUtils; - - - import org.apache.usergrid.management.OrganizationInfo; ++import com.sun.jersey.api.client.ClientResponse.Status; ++import com.sun.jersey.api.client.UniformInterfaceException; ++import com.sun.jersey.api.representation.Form; + import org.apache.commons.lang.RandomStringUtils; -import org.codehaus.jackson.JsonNode; -import org.junit.Test; - -import org.apache.commons.lang.StringUtils; - -import org.apache.usergrid.cassandra.Concurrent; -import org.apache.usergrid.management.OrganizationInfo; import org.apache.usergrid.management.OrganizationOwnerInfo; +import org.apache.usergrid.persistence.index.utils.UUIDUtils; import org.apache.usergrid.rest.AbstractRestIT; - import org.apache.usergrid.rest.TestContextSetup; import org.apache.usergrid.rest.management.organizations.OrganizationsResource; - -import com.sun.jersey.api.client.ClientResponse.Status; -import com.sun.jersey.api.client.UniformInterfaceException; -import com.sun.jersey.api.representation.Form; ++import org.junit.Test; + import org.slf4j.Logger; + import org.slf4j.LoggerFactory; - import com.sun.jersey.api.client.ClientResponse.Status; - import com.sun.jersey.api.client.UniformInterfaceException; - import com.sun.jersey.api.representation.Form; - -import static org.apache.usergrid.utils.MapUtils.hashMap; ++import javax.ws.rs.core.MediaType; +import java.io.IOException; ++import java.util.*; + import static org.apache.usergrid.rest.management.ManagementResource.USERGRID_CENTRAL_URL; +import static org.apache.usergrid.utils.MapUtils.hashMap; - import static org.junit.Assert.assertEquals; - import static org.junit.Assert.assertNotNull; - import static org.junit.Assert.assertNull; - import static org.junit.Assert.assertTrue; + import static org.junit.Assert.*; /** * @author tnine */ -@Concurrent() + public class ManagementResourceIT extends AbstractRestIT { - @Rule - public TestContextSetup context = new TestContextSetup( this ); + private static final Logger logger = LoggerFactory.getLogger(ManagementResourceIT.class); + public ManagementResourceIT() throws Exception { } /** + * Test if we can reset our password as an admin + */ + @Test + public void setSelfAdminPasswordAsAdmin() { + + String newPassword = "foo"; + + Map<String, String> data = new HashMap<String, String>(); + data.put( "newpassword", newPassword ); + data.put( "oldpassword", "test" ); + + // change the password as admin. The old password isn't required + JsonNode node = resource().path( "/management/users/test/password" ).accept( MediaType.APPLICATION_JSON ) + .type( MediaType.APPLICATION_JSON_TYPE ).post( JsonNode.class, data ); + + assertNull( getError( node ) ); + + adminAccessToken = mgmtToken( "test", newPassword ); + + data.put( "oldpassword", newPassword ); + data.put( "newpassword", "test" ); + + node = resource().path( "/management/users/test/password" ).queryParam( "access_token", adminAccessToken ) + .accept( MediaType.APPLICATION_JSON ).type( MediaType.APPLICATION_JSON_TYPE ) + .post( JsonNode.class, data ); + + assertNull( getError( node ) ); + } + + - @Test - public void passwordMismatchErrorAdmin() { - String origPassword = "foo"; - String newPassword = "bar"; - - Map<String, String> data = new HashMap<String, String>(); - data.put( "newpassword", origPassword ); - - // now change the password, with an incorrect old password - - data.put( "oldpassword", origPassword ); - data.put( "newpassword", newPassword ); - - Status responseStatus = null; - - try { - resource().path( "/management/users/test/password" ).accept( MediaType.APPLICATION_JSON ) - .type( MediaType.APPLICATION_JSON_TYPE ).post( JsonNode.class, data ); - } - catch ( UniformInterfaceException uie ) { - responseStatus = uie.getResponse().getClientResponseStatus(); - } - - assertNotNull( responseStatus ); - - assertEquals( Status.BAD_REQUEST, responseStatus ); - } - - - @Test - public void setAdminPasswordAsSysAdmin() { - - String superToken = superAdminToken(); - - String newPassword = "foo"; - - Map<String, String> data = new HashMap<String, String>(); - data.put( "newpassword", newPassword ); - - // change the password as admin. The old password isn't required - JsonNode node = resource().path( "/management/users/test/password" ).queryParam( "access_token", superToken ) - .accept( MediaType.APPLICATION_JSON ).type( MediaType.APPLICATION_JSON_TYPE ) - .post( JsonNode.class, data ); - - assertNull( getError( node ) ); - - // log in with the new password - String token = mgmtToken( "test", newPassword ); - - assertNotNull( token ); - - data.put( "newpassword", "test" ); - - // now change the password back - node = resource().path( "/management/users/test/password" ).queryParam( "access_token", superToken ) - .accept( MediaType.APPLICATION_JSON ).type( MediaType.APPLICATION_JSON_TYPE ) - .post( JsonNode.class, data ); - - assertNull( getError( node ) ); - } - + + /** * Test that admins can't view organizations they're not authorized to view. */ @Test public void crossOrgsNotViewable() throws Exception { - OrganizationOwnerInfo orgInfo = setup.getMgmtSvc().createOwnerAndOrganization( "crossOrgsNotViewable", - "crossOrgsNotViewable", "TestName", "crossorgsnotviewa...@usergrid.org", "password" ); + String username = "test" + UUIDUtils.newTimeUUID(); + String name = username; + String email = username + "@usergrid.com"; + String password = "password"; + String orgName = username; - // check that the test admin cannot access the new org info - - Status status = null; + Map payload = + hashMap( "email", email ).map( "username", username ).map( "name", name ).map( "password", password ) + .map( "organization", orgName ).map( "company", "Apigee" ); - try { - resource().path( String.format( "/management/orgs/%s", orgInfo.getOrganization().getName() ) ) - .queryParam( "access_token", adminAccessToken ).accept( MediaType.APPLICATION_JSON ) - .type( MediaType.APPLICATION_JSON_TYPE ).get( JsonNode.class ); - } - catch ( UniformInterfaceException uie ) { - status = uie.getResponse().getClientResponseStatus(); - } + JsonNode node = mapper.readTree( + resource().path( "/management/organizations" ).accept( MediaType.APPLICATION_JSON ) + .type( MediaType.APPLICATION_JSON_TYPE ).post( String.class, payload ) ); - assertNotNull( status ); - assertEquals( Status.UNAUTHORIZED, status ); + // check that the test admin cannot access the new org info - status = null; + Status status = null; try { - resource().path( String.format( "/management/orgs/%s", orgInfo.getOrganization().getUuid() ) ) - .queryParam( "access_token", adminAccessToken ).accept( MediaType.APPLICATION_JSON ) - .type( MediaType.APPLICATION_JSON_TYPE ).get( JsonNode.class ); ++ this. + resource().path( String.format( "/management/orgs/%s", orgName ) ) - .queryParam( "access_token", context.getActiveUser().getToken() ) ++ .queryParam( "access_token", this.adminToken() ) + .accept( MediaType.APPLICATION_JSON ).type( MediaType.APPLICATION_JSON_TYPE ).get( String.class ); } catch ( UniformInterfaceException uie ) { status = uie.getResponse().getClientResponseStatus(); @@@ -108,9 -198,9 +125,9 @@@ // this admin should have access to test org status = null; try { - resource().path( "/management/orgs/" + context.getOrgName() ) - .queryParam( "access_token", context.getActiveUser().getToken() ) - resource().path( "/management/orgs/test-organization" ).queryParam( "access_token", adminAccessToken ) - .accept( MediaType.APPLICATION_JSON ).type( MediaType.APPLICATION_JSON_TYPE ) - .get( JsonNode.class ); ++ resource().path( "/management/orgs/" + this.orgInfo.getName()) ++ .queryParam( "access_token", this.adminToken() ) + .accept( MediaType.APPLICATION_JSON ).type( MediaType.APPLICATION_JSON_TYPE ).get( String.class ); } catch ( UniformInterfaceException uie ) { status = uie.getResponse().getClientResponseStatus(); @@@ -122,9 -212,9 +139,9 @@@ status = null; try { - resource().path( String.format( "/management/orgs/%s", context.getOrgUuid() ) ) - .queryParam( "access_token", context.getActiveUser().getToken() ) - resource().path( String.format( "/management/orgs/%s", org.getUuid() ) ) - .queryParam( "access_token", adminAccessToken ).accept( MediaType.APPLICATION_JSON ) - .type( MediaType.APPLICATION_JSON_TYPE ).get( JsonNode.class ); ++ resource().path( String.format( "/management/orgs/%s", this.orgInfo.getUuid() ) ) ++ .queryParam( "access_token", this.adminToken() ) + .accept( MediaType.APPLICATION_JSON ).type( MediaType.APPLICATION_JSON_TYPE ).get( String.class ); } catch ( UniformInterfaceException uie ) { status = uie.getResponse().getClientResponseStatus(); @@@ -145,9 -245,6 +162,9 @@@ for ( i = 0; i < 10; i++ ) { users1.add( "follower" + Integer.toString( i ) ); } + - refreshIndex( context.getOrgName(), context.getAppName() ); ++ refreshIndex( this.orgInfo.getName(), this.appInfo.getName() ); + checkFeed( "leader1", users1 ); //try with 11 List<String> users2 = new ArrayList<String>(); @@@ -158,25 -255,17 +175,25 @@@ } - private void checkFeed( String leader, List<String> followers ) { + private void checkFeed( String leader, List<String> followers ) throws IOException { JsonNode userFeed; + //create user createUser( leader ); - refreshIndex( context.getOrgName(), context.getAppName() ); ++ refreshIndex( this.orgInfo.getName(), this.appInfo.getName() ); + String preFollowContent = leader + ": pre-something to look for " + UUID.randomUUID().toString(); + addActivity( leader, leader + " " + leader + "son", preFollowContent ); - refreshIndex( context.getOrgName(), context.getAppName() ); ++ refreshIndex( this.orgInfo.getName(), this.appInfo.getName() ); + String lastUser = followers.get( followers.size() - 1 ); int i = 0; for ( String user : followers ) { createUser( user ); - refreshIndex( context.getOrgName(), context.getAppName() ); ++ refreshIndex( this.orgInfo.getName(), this.appInfo.getName() ); follow( user, leader ); - refreshIndex( context.getOrgName(), context.getAppName() ); ++ refreshIndex( this.orgInfo.getName(), this.appInfo.getName() ); } userFeed = getUserFeed( lastUser ); assertTrue( userFeed.size() == 1 ); @@@ -186,9 -275,6 +203,9 @@@ assertTrue( userFeed.size() == 1 ); String postFollowContent = leader + ": something to look for " + UUID.randomUUID().toString(); addActivity( leader, leader + " " + leader + "son", postFollowContent ); + - refreshIndex( context.getOrgName(), context.getAppName() ); ++ refreshIndex( this.orgInfo.getName(), this.appInfo.getName() ); + //check feed userFeed = getUserFeed( lastUser ); assertNotNull( userFeed ); @@@ -202,28 -288,25 +219,28 @@@ private void createUser( String username ) { Map<String, Object> payload = new LinkedHashMap<String, Object>(); payload.put( "username", username ); - resource().path( "" + context.getOrgName() + "/" + context.getAppName() + "/users" ) - .queryParam( "access_token", context.getActiveUser().getToken() ).accept( MediaType.APPLICATION_JSON ) - resource().path( "/test-organization/test-app/users" ).queryParam( "access_token", access_token ) - .accept( MediaType.APPLICATION_JSON ).type( MediaType.APPLICATION_JSON_TYPE ) - .post( JsonNode.class, payload ); ++ resource().path( "" + orgInfo.getName() + "/" + appInfo.getName() + "/users" ) ++ .queryParam( "access_token", this.adminToken() ).accept( MediaType.APPLICATION_JSON ) + .type( MediaType.APPLICATION_JSON_TYPE ).post( String.class, payload ); } - private JsonNode getUserFeed( String username ) { - JsonNode userFeed = resource().path( "/test-organization/test-app/users/" + username + "/feed" ) - .queryParam( "access_token", access_token ).accept( MediaType.APPLICATION_JSON ) - .get( JsonNode.class ); + private JsonNode getUserFeed( String username ) throws IOException { + JsonNode userFeed = mapper.readTree( resource() - .path( "/" + context.getOrgName() + "/" + context.getAppName() + "/users/" + username + "/feed" ) - .queryParam( "access_token", context.getActiveUser().getToken() ).accept( MediaType.APPLICATION_JSON ) ++ .path( "/" + orgInfo.getName() + "/" + appInfo.getName() + "/users/" + username + "/feed" ) ++ .queryParam( "access_token", this.adminToken() ).accept( MediaType.APPLICATION_JSON ) + .get( String.class ) ); return userFeed.get( "entities" ); } private void follow( String user, String followUser ) { //post follow - resource().path( "/test-organization/test-app/users/" + user + "/following/users/" + followUser ) - .queryParam( "access_token", access_token ).accept( MediaType.APPLICATION_JSON ) - .type( MediaType.APPLICATION_JSON_TYPE ).post( JsonNode.class, new HashMap<String, String>() ); + resource() - .path( "/" + context.getOrgName() + "/" + context.getAppName() + "/users/" + user + "/following/users/" - + followUser ).queryParam( "access_token", context.getActiveUser().getToken() ) ++ .path( "/" + orgInfo.getName() + "/" + appInfo.getName() + "/users/" + user + "/following/users/" ++ + followUser ).queryParam( "access_token", this.adminToken() ) + .accept( MediaType.APPLICATION_JSON ).type( MediaType.APPLICATION_JSON_TYPE ) + .post( String.class, new HashMap<String, String>() ); } @@@ -235,9 -318,9 +252,9 @@@ actorMap.put( "displayName", name ); actorMap.put( "username", user ); activityPayload.put( "actor", actorMap ); - resource().path( "/" + context.getOrgName() + "/" + context.getAppName() + "/users/" + user + "/activities" ) - .queryParam( "access_token", context.getActiveUser().getToken() ).accept( MediaType.APPLICATION_JSON ) - resource().path( "/test-organization/test-app/users/" + user + "/activities" ) - .queryParam( "access_token", access_token ).accept( MediaType.APPLICATION_JSON ) - .type( MediaType.APPLICATION_JSON_TYPE ).post( JsonNode.class, activityPayload ); ++ resource().path( "/" + orgInfo.getName() + "/" + appInfo.getName() + "/users/" + user + "/activities" ) ++ .queryParam( "access_token", this.adminToken() ).accept( MediaType.APPLICATION_JSON ) + .type( MediaType.APPLICATION_JSON_TYPE ).post( String.class, activityPayload ); } @@@ -247,45 -330,462 +264,475 @@@ Map<String, String> data = new HashMap<String, String>(); data.put( "name", "mgmt-org-app" ); - String orgName = context.getOrgName(); ++ String orgName = orgInfo.getName(); + // POST /applications - JsonNode appdata = resource().path( "/management/orgs/" + orgInfo.getUuid() + "/applications" ) - .queryParam( "access_token", adminToken() ).accept( MediaType.APPLICATION_JSON ) - .type( MediaType.APPLICATION_JSON_TYPE ).post( JsonNode.class, data ); + JsonNode appdata = mapper.readTree( resource().path( "/management/orgs/" + orgName + "/applications" ) - .queryParam( "access_token", context.getActiveUser().getToken() ) ++ .queryParam( "access_token", this.adminToken() ) + .accept( MediaType.APPLICATION_JSON ) + .type( MediaType.APPLICATION_JSON_TYPE ) + .post( String.class, data ) ); logNode( appdata ); appdata = getEntity( appdata, 0 ); - refreshIndex( orgName, context.getAppName() ); - assertEquals( "test-organization/mgmt-org-app", appdata.get( "name" ).asText() ); ++ refreshIndex( this.orgInfo.getName(), this.appInfo.getName() ); + + assertEquals( orgName.toLowerCase() + "/mgmt-org-app", appdata.get( "name" ).asText() ); + assertNotNull( appdata.get( "metadata" ) ); + assertNotNull( appdata.get( "metadata" ).get( "collections" ) ); + assertNotNull( appdata.get( "metadata" ).get( "collections" ).get( "roles" ) ); + assertNotNull( appdata.get( "metadata" ).get( "collections" ).get( "roles" ).get( "title" ) ); assertEquals( "Roles", appdata.get( "metadata" ).get( "collections" ).get( "roles" ).get( "title" ).asText() ); assertEquals( 3, appdata.get( "metadata" ).get( "collections" ).get( "roles" ).get( "count" ).asInt() ); - refreshIndex( orgName, context.getAppName() ); ++ refreshIndex( this.orgInfo.getName(), this.appInfo.getName() ); + // GET /applications/mgmt-org-app - appdata = resource().path( "/management/orgs/" + orgInfo.getUuid() + "/applications/mgmt-org-app" ) - .queryParam( "access_token", adminToken() ).accept( MediaType.APPLICATION_JSON ) - .type( MediaType.APPLICATION_JSON_TYPE ).get( JsonNode.class ); + appdata = mapper.readTree( - resource().path( "/management/orgs/" + context.getOrgUuid() + "/applications/mgmt-org-app" ) - .queryParam( "access_token", context.getActiveUser().getToken() ) ++ resource().path( "/management/orgs/" + orgInfo.getUuid() + "/applications/mgmt-org-app" ) ++ .queryParam( "access_token", this.adminToken() ) + .accept( MediaType.APPLICATION_JSON ).type( MediaType.APPLICATION_JSON_TYPE ) + .get( String.class ) ); logNode( appdata ); - assertEquals( "test-organization", appdata.get( "organization" ).asText() ); + assertEquals( orgName.toLowerCase(), appdata.get( "organization" ).asText() ); assertEquals( "mgmt-org-app", appdata.get( "applicationName" ).asText() ); - assertEquals( "http://sometestvalue/test-organization/mgmt-org-app";, appdata.get( "uri" ).getTextValue() ); + assertEquals( "http://sometestvalue/"; + orgName.toLowerCase() + "/mgmt-org-app", + appdata.get( "uri" ).textValue() ); appdata = getEntity( appdata, 0 ); - assertEquals( "test-organization/mgmt-org-app", appdata.get( "name" ).asText() ); + assertEquals( orgName.toLowerCase() + "/mgmt-org-app", appdata.get( "name" ).asText() ); assertEquals( "Roles", appdata.get( "metadata" ).get( "collections" ).get( "roles" ).get( "title" ).asText() ); assertEquals( 3, appdata.get( "metadata" ).get( "collections" ).get( "roles" ).get( "count" ).asInt() ); } + - + @Test + public void tokenTtl() throws Exception { + + long ttl = 2000; + + JsonNode node = resource().path( "/management/token" ).queryParam( "grant_type", "password" ) + .queryParam( "username", "t...@usergrid.com" ).queryParam( "password", "test" ) + .queryParam( "ttl", String.valueOf( ttl ) ).accept( MediaType.APPLICATION_JSON ) + .get( JsonNode.class ); + + long startTime = System.currentTimeMillis(); + - String token = node.get( "access_token" ).getTextValue(); ++ String token = node.get( "access_token" ).textValue(); + + assertNotNull( token ); + + JsonNode userdata = resource().path( "/management/users/t...@usergrid.com" ).queryParam( "access_token", token ) + .accept( MediaType.APPLICATION_JSON ).get( JsonNode.class ); + + assertEquals( "t...@usergrid.com", userdata.get( "data" ).get( "email" ).asText() ); + + // wait for the token to expire + Thread.sleep( ttl - (System.currentTimeMillis() - startTime) + 1000 ); + + Status responseStatus = null; + try { + userdata = resource().path( "/management/users/t...@usergrid.com" ).accept( MediaType.APPLICATION_JSON ) + .type( MediaType.APPLICATION_JSON_TYPE ).get( JsonNode.class ); + } + catch ( UniformInterfaceException uie ) { + responseStatus = uie.getResponse().getClientResponseStatus(); + } + + assertEquals( Status.UNAUTHORIZED, responseStatus ); + } + + + @Test + public void token() throws Exception { + JsonNode node = resource().path( "/management/token" ).queryParam( "grant_type", "password" ) + .queryParam( "username", "t...@usergrid.com" ).queryParam( "password", "test" ) + .accept( MediaType.APPLICATION_JSON ).get( JsonNode.class ); + + logNode( node ); - String token = node.get( "access_token" ).getTextValue(); ++ String token = node.get( "access_token" ).textValue(); + assertNotNull( token ); + + // set an organization property + HashMap<String, Object> payload = new HashMap<String, Object>(); + Map<String, Object> properties = new HashMap<String, Object>(); + properties.put( "securityLevel", 5 ); + payload.put( OrganizationsResource.ORGANIZATION_PROPERTIES, properties ); + node = resource().path( "/management/organizations/test-organization" ) + .queryParam( "access_token", superAdminToken() ).accept( MediaType.APPLICATION_JSON ) + .type( MediaType.APPLICATION_JSON_TYPE ).put( JsonNode.class, payload ); + + // ensure the organization property is included + node = resource().path( "/management/token" ).queryParam( "access_token", token ) + .accept( MediaType.APPLICATION_JSON ).get( JsonNode.class ); + logNode( node ); + + JsonNode securityLevel = node.findValue( "securityLevel" ); + assertNotNull( securityLevel ); + assertEquals( 5L, securityLevel.asLong() ); + } + + + @Test + public void meToken() throws Exception { + JsonNode node = resource().path( "/management/me" ).queryParam( "grant_type", "password" ) + .queryParam( "username", "t...@usergrid.com" ).queryParam( "password", "test" ) + .accept( MediaType.APPLICATION_JSON ).get( JsonNode.class ); + + logNode( node ); - String token = node.get( "access_token" ).getTextValue(); ++ String token = node.get( "access_token" ).textValue(); + assertNotNull( token ); + + node = resource().path( "/management/me" ).queryParam( "access_token", token ) + .accept( MediaType.APPLICATION_JSON ).get( JsonNode.class ); + logNode( node ); + + assertNotNull( node.get( "passwordChanged" ) ); + assertNotNull( node.get( "access_token" ) ); + assertNotNull( node.get( "expires_in" ) ); + JsonNode userNode = node.get( "user" ); + assertNotNull( userNode ); + assertNotNull( userNode.get( "uuid" ) ); + assertNotNull( userNode.get( "username" ) ); + assertNotNull( userNode.get( "email" ) ); + assertNotNull( userNode.get( "name" ) ); + assertNotNull( userNode.get( "properties" ) ); + JsonNode orgsNode = userNode.get( "organizations" ); + assertNotNull( orgsNode ); + JsonNode orgNode = orgsNode.get( "test-organization" ); + assertNotNull( orgNode ); + assertNotNull( orgNode.get( "name" ) ); + assertNotNull( orgNode.get( "properties" ) ); + } + + + @Test + public void meTokenPost() throws Exception { + Map<String, String> payload = + hashMap( "grant_type", "password" ).map( "username", "t...@usergrid.com" ).map( "password", "test" ); + + JsonNode node = resource().path( "/management/me" ).accept( MediaType.APPLICATION_JSON ) + .type( MediaType.APPLICATION_JSON_TYPE ).post( JsonNode.class, payload ); + + logNode( node ); - String token = node.get( "access_token" ).getTextValue(); ++ String token = node.get( "access_token" ).textValue(); + + assertNotNull( token ); + + node = resource().path( "/management/me" ).queryParam( "access_token", token ) + .accept( MediaType.APPLICATION_JSON ).get( JsonNode.class ); + logNode( node ); + } + + + @Test + public void meTokenPostForm() { + + Form form = new Form(); + form.add( "grant_type", "password" ); + form.add( "username", "t...@usergrid.com" ); + form.add( "password", "test" ); + + JsonNode node = resource().path( "/management/me" ).accept( MediaType.APPLICATION_JSON ) + .type( MediaType.APPLICATION_FORM_URLENCODED_TYPE ) + .entity( form, MediaType.APPLICATION_FORM_URLENCODED_TYPE ).post( JsonNode.class ); + + logNode( node ); - String token = node.get( "access_token" ).getTextValue(); ++ String token = node.get( "access_token" ).textValue(); + + assertNotNull( token ); + + node = resource().path( "/management/me" ).queryParam( "access_token", token ) + .accept( MediaType.APPLICATION_JSON ).get( JsonNode.class ); + logNode( node ); + } + + + @Test + public void ttlNan() throws Exception { + + Map<String, String> payload = + hashMap( "grant_type", "password" ).map( "username", "t...@usergrid.com" ).map( "password", "test" ) + .map( "ttl", "derp" ); + + Status responseStatus = null; + try { + resource().path( "/management/token" ).accept( MediaType.APPLICATION_JSON ) + .type( MediaType.APPLICATION_JSON_TYPE ).post( JsonNode.class, payload ); + } + catch ( UniformInterfaceException uie ) { + responseStatus = uie.getResponse().getClientResponseStatus(); + } + + assertEquals( Status.BAD_REQUEST, responseStatus ); + } + + + @Test + public void ttlOverMax() throws Exception { + + Map<String, String> payload = + hashMap( "grant_type", "password" ).map( "username", "t...@usergrid.com" ).map( "password", "test" ) + .map( "ttl", Long.MAX_VALUE + "" ); + + Status responseStatus = null; + + try { + resource().path( "/management/token" ).accept( MediaType.APPLICATION_JSON ) + .type( MediaType.APPLICATION_JSON_TYPE ).post( JsonNode.class, payload ); + } + catch ( UniformInterfaceException uie ) { + responseStatus = uie.getResponse().getClientResponseStatus(); + } + + assertEquals( Status.BAD_REQUEST, responseStatus ); + } + + + @Test + public void revokeToken() throws Exception { + String token1 = super.adminToken(); + String token2 = super.adminToken(); + + JsonNode response = resource().path( "/management/users/test" ).queryParam( "access_token", token1 ) + .accept( MediaType.APPLICATION_JSON ).type( MediaType.APPLICATION_JSON_TYPE ) + .get( JsonNode.class ); + + assertEquals( "t...@usergrid.com", response.get( "data" ).get( "email" ).asText() ); + + response = resource().path( "/management/users/test" ).queryParam( "access_token", token2 ) + .accept( MediaType.APPLICATION_JSON ).type( MediaType.APPLICATION_JSON_TYPE ) + .get( JsonNode.class ); + + assertEquals( "t...@usergrid.com", response.get( "data" ).get( "email" ).asText() ); + + // now revoke the tokens + response = + resource().path( "/management/users/test/revoketokens" ).queryParam( "access_token", superAdminToken() ) + .accept( MediaType.APPLICATION_JSON ).type( MediaType.APPLICATION_JSON_TYPE ) + .post( JsonNode.class ); + + // the tokens shouldn't work + + Status status = null; + + try { + response = resource().path( "/management/users/test" ).queryParam( "access_token", token1 ) + .accept( MediaType.APPLICATION_JSON ).type( MediaType.APPLICATION_JSON_TYPE ) + .get( JsonNode.class ); + } + catch ( UniformInterfaceException uie ) { + status = uie.getResponse().getClientResponseStatus(); + } + + assertEquals( Status.UNAUTHORIZED, status ); + + status = null; + + try { + response = resource().path( "/management/users/test" ).queryParam( "access_token", token2 ) + .accept( MediaType.APPLICATION_JSON ).type( MediaType.APPLICATION_JSON_TYPE ) + .get( JsonNode.class ); + } + catch ( UniformInterfaceException uie ) { + status = uie.getResponse().getClientResponseStatus(); + } + + assertEquals( Status.UNAUTHORIZED, status ); + + String token3 = super.adminToken(); + String token4 = super.adminToken(); + + response = resource().path( "/management/users/test" ).queryParam( "access_token", token3 ) + .accept( MediaType.APPLICATION_JSON ).type( MediaType.APPLICATION_JSON_TYPE ) + .get( JsonNode.class ); + + assertEquals( "t...@usergrid.com", response.get( "data" ).get( "email" ).asText() ); + + response = resource().path( "/management/users/test" ).queryParam( "access_token", token4 ) + .accept( MediaType.APPLICATION_JSON ).type( MediaType.APPLICATION_JSON_TYPE ) + .get( JsonNode.class ); + + assertEquals( "t...@usergrid.com", response.get( "data" ).get( "email" ).asText() ); + + // now revoke the token3 + response = resource().path( "/management/users/test/revoketoken" ).queryParam( "access_token", token3 ) + .queryParam( "token", token3 ).accept( MediaType.APPLICATION_JSON ) + .type( MediaType.APPLICATION_JSON_TYPE ).post( JsonNode.class ); + + // the token3 shouldn't work + + status = null; + + try { + response = resource().path( "/management/users/test" ).queryParam( "access_token", token3 ) + .accept( MediaType.APPLICATION_JSON ).type( MediaType.APPLICATION_JSON_TYPE ) + .get( JsonNode.class ); + } + catch ( UniformInterfaceException uie ) { + status = uie.getResponse().getClientResponseStatus(); + } + + assertEquals( Status.UNAUTHORIZED, status ); + + status = null; + + try { + response = resource().path( "/management/users/test" ).queryParam( "access_token", token4 ) + .accept( MediaType.APPLICATION_JSON ).type( MediaType.APPLICATION_JSON_TYPE ) + .get( JsonNode.class ); + + status = Status.OK; + } + catch ( UniformInterfaceException uie ) { + status = uie.getResponse().getClientResponseStatus(); + } + + assertEquals( Status.OK, status ); + } + + + @Test + public void testValidateExternalToken() throws Exception { + + // create a new admin user, get access token + + String rand = RandomStringUtils.randomAlphanumeric(10); + final String username = "user_" + rand; + OrganizationOwnerInfo orgInfo = setup.getMgmtSvc().createOwnerAndOrganization( + username, username, "Test User", username + "@example.com", "password" ); + + Map<String, Object> loginInfo = new HashMap<String, Object>() {{ + put("username", username ); + put("password", "password"); + put("grant_type", "password"); + }}; + JsonNode accessInfoNode = resource().path("/management/token") + .type( MediaType.APPLICATION_JSON_TYPE ) + .post( JsonNode.class, loginInfo ); - String accessToken = accessInfoNode.get( "access_token" ).getTextValue(); ++ String accessToken = accessInfoNode.get( "access_token" ).textValue(); + + // set the Usergrid Central SSO URL because Tomcat port is dynamically assigned + + String suToken = superAdminToken(); + Map<String, String> props = new HashMap<String, String>(); + props.put( USERGRID_CENTRAL_URL, getBaseURI().toURL().toExternalForm() ); + resource().path( "/testproperties" ) + .queryParam( "access_token", suToken) + .accept( MediaType.APPLICATION_JSON ) + .type( MediaType.APPLICATION_JSON_TYPE ) + .post( props ); + + // attempt to validate the token, must be valid + + JsonNode validatedNode = resource().path( "/management/externaltoken" ) + .queryParam( "access_token", suToken ) // as superuser + .queryParam( "ext_access_token", accessToken ) + .queryParam( "ttl", "1000" ) + .get( JsonNode.class ); - String validatedAccessToken = validatedNode.get( "access_token" ).getTextValue(); ++ String validatedAccessToken = validatedNode.get( "access_token" ).textValue(); + assertEquals( accessToken, validatedAccessToken ); + + // attempt to validate an invalid token, must fail + + try { + resource().path( "/management/externaltoken" ) + .queryParam( "access_token", suToken ) // as superuser + .queryParam( "ext_access_token", "rubbish_token") + .queryParam( "ttl", "1000" ) + .get( JsonNode.class ); + fail("Validation should have failed"); + } catch ( UniformInterfaceException actual ) { + assertEquals( 404, actual.getResponse().getStatus() ); + String errorMsg = actual.getResponse().getEntity( JsonNode.class ).get( "error_description" ).toString(); + logger.error( "ERROR: " + errorMsg ); + assertTrue( errorMsg.contains( "Cannot find Admin User" ) ); + } + + + + // TODO: how do we test the create new user and organization case? + + + + // unset the Usergrid Central SSO URL so it does not interfere with other tests + + props.put( USERGRID_CENTRAL_URL, "" ); + resource().path( "/testproperties" ) + .queryParam( "access_token", suToken) + .accept( MediaType.APPLICATION_JSON ) + .type( MediaType.APPLICATION_JSON_TYPE ) + .post( props ); + + } + + + @Test + public void testSuperuserOnlyWhenValidateExternalTokensEnabled() throws Exception { + + // create an org and an admin user + + String rand = RandomStringUtils.randomAlphanumeric( 10 ); + final String username = "user_" + rand; + OrganizationOwnerInfo orgInfo = setup.getMgmtSvc().createOwnerAndOrganization( + username, username, "Test User", username + "@example.com", "password" ); + + // turn on validate external tokens by setting the usergrid.central.url + + String suToken = superAdminToken(); + Map<String, String> props = new HashMap<String, String>(); + props.put( USERGRID_CENTRAL_URL, getBaseURI().toURL().toExternalForm()); + resource().path( "/testproperties" ) + .queryParam( "access_token", suToken) + .accept( MediaType.APPLICATION_JSON ) + .type( MediaType.APPLICATION_JSON_TYPE ) + .post( props ); + + // calls to login as an Admin User must now fail + + try { + + Map<String, Object> loginInfo = new HashMap<String, Object>() {{ + put("username", username ); + put("password", "password"); + put("grant_type", "password"); + }}; + JsonNode accessInfoNode = resource().path("/management/token") + .type( MediaType.APPLICATION_JSON_TYPE ) + .post( JsonNode.class, loginInfo ); + fail("Login as Admin User must fail when validate external tokens is enabled"); + + } catch ( UniformInterfaceException actual ) { + assertEquals( 400, actual.getResponse().getStatus() ); + String errorMsg = actual.getResponse().getEntity( JsonNode.class ).get( "error_description" ).toString(); + logger.error( "ERROR: " + errorMsg ); + assertTrue( errorMsg.contains( "Admin Users must login via" )); + + } catch ( Exception e ) { + fail( "We expected a UniformInterfaceException" ); + } + + // login as superuser must succeed + + Map<String, Object> loginInfo = new HashMap<String, Object>() {{ + put("username", "superuser"); + put("password", "superpassword"); + put("grant_type", "password"); + }}; + JsonNode accessInfoNode = resource().path("/management/token") + .type( MediaType.APPLICATION_JSON_TYPE ) + .post( JsonNode.class, loginInfo ); - String accessToken = accessInfoNode.get( "access_token" ).getTextValue(); ++ String accessToken = accessInfoNode.get( "access_token" ).textValue(); + assertNotNull( accessToken ); + + // turn off validate external tokens by un-setting the usergrid.central.url + + props.put( USERGRID_CENTRAL_URL, "" ); + resource().path( "/testproperties" ) + .queryParam( "access_token", suToken) + .accept( MediaType.APPLICATION_JSON ) + .type( MediaType.APPLICATION_JSON_TYPE ) + .post( props ); + } + }
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/a7840164/stack/rest/src/test/java/org/apache/usergrid/rest/test/resource2point0/AbstractRestIT.java ---------------------------------------------------------------------- diff --cc stack/rest/src/test/java/org/apache/usergrid/rest/test/resource2point0/AbstractRestIT.java index 4e3b480,0000000..9ae5e3b mode 100644,000000..100644 --- a/stack/rest/src/test/java/org/apache/usergrid/rest/test/resource2point0/AbstractRestIT.java +++ b/stack/rest/src/test/java/org/apache/usergrid/rest/test/resource2point0/AbstractRestIT.java @@@ -1,179 -1,0 +1,177 @@@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.usergrid.rest.test.resource2point0; + + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.sun.jersey.api.client.UniformInterfaceException; +import com.sun.jersey.api.client.config.ClientConfig; +import com.sun.jersey.api.client.config.DefaultClientConfig; +import com.sun.jersey.api.json.JSONConfiguration; +import com.sun.jersey.test.framework.AppDescriptor; +import com.sun.jersey.test.framework.JerseyTest; +import com.sun.jersey.test.framework.WebAppDescriptor; +import com.sun.jersey.test.framework.spi.container.TestContainerFactory; +import org.apache.usergrid.rest.TomcatRuntime; +import org.apache.usergrid.rest.test.resource2point0.endpoints.ApplicationsResource; +import org.apache.usergrid.rest.test.resource2point0.endpoints.OrganizationResource; +import org.apache.usergrid.rest.test.resource2point0.endpoints.mgmt.ManagementResource; +import org.apache.usergrid.rest.test.resource2point0.model.Token; +import org.apache.usergrid.rest.test.resource2point0.state.ClientContext; +import org.junit.Rule; + +import java.net.URI; +import java.net.URISyntaxException; +import java.net.URLClassLoader; +import java.util.Arrays; + +import static org.junit.Assert.assertEquals; + + + +/** + * Base class for REST tests. + */ +//@RunWith( Arquillian.class ) +public class AbstractRestIT extends JerseyTest { + + private static ClientConfig clientConfig = new DefaultClientConfig(); + + public static TomcatRuntime tomcatRuntime = TomcatRuntime.getInstance(); + - - + @Rule + public ClientSetup clientSetup = new ClientSetup( this.getBaseURI().toString() ); + + protected static final AppDescriptor descriptor; + + public AbstractRestIT() { + super( descriptor ); + } + + + protected ObjectMapper mapper = new ObjectMapper(); + + static { + clientConfig.getFeatures().put( JSONConfiguration.FEATURE_POJO_MAPPING, Boolean.TRUE ); + descriptor = new WebAppDescriptor.Builder( "org.apache.usergrid.rest" ) + .clientConfig( clientConfig ).build(); + dumpClasspath( AbstractRestIT.class.getClassLoader() ); + } + + +// //We set testable = false so we deploy the archive to the server and test it locally +// @Deployment( testable = false ) +// public static WebArchive createTestArchive() { +// +// //we use the MavenImporter from shrinkwrap to just produce whatever maven would build then test with it +// +// //set maven to be in offline mode +// +// System.setProperty( "org.apache.maven.offline", "true" ); +// +// return ShrinkWrap.create( MavenImporter.class ).loadPomFromFile( "pom.xml", "arquillian-tomcat" ) +// .importBuildOutput().as( WebArchive.class ); +// } + + public static void dumpClasspath( ClassLoader loader ) { + System.out.println( "Classloader " + loader + ":" ); + + if ( loader instanceof URLClassLoader ) { + URLClassLoader ucl = ( URLClassLoader ) loader; + System.out.println( "\t" + Arrays.toString( ucl.getURLs() ) ); + } + else { + System.out.println( "\t(cannot display components as not a URLClassLoader)" ); + } + + if ( loader.getParent() != null ) { + dumpClasspath( loader.getParent() ); + } + } + + @Override + protected URI getBaseURI() { + try { + return new URI("http://localhost:"; + tomcatRuntime.getPort()); + } catch (URISyntaxException e) { + throw new RuntimeException("Error determining baseURI", e); + } + } + + @Override + protected TestContainerFactory getTestContainerFactory() { + return new com.sun.jersey.test.framework.spi.container.external.ExternalTestContainerFactory(); + } + + ///myorg/ + protected OrganizationResource org(){ + return clientSetup.restClient.org( clientSetup.getOrganization().getName() ); + } + + //myorg/myapp + protected ApplicationsResource app(){ + return clientSetup.restClient.org(clientSetup.getOrganization().getName()).app(clientSetup.getAppName()); + + } + + protected ManagementResource management(){ + return clientSetup.restClient.management(); + } + + protected ClientContext context(){ + return this.clientSetup.getRestClient().getContext(); + } + + + protected Token getAppUserToken(String username, String password){ + return this.app().token().post(new Token(username,password)); + } + + public void refreshIndex() { + //TODO: add error checking and logging + clientSetup.refreshIndex(); + } + + + /** + * Takes in the expectedStatus message and the expectedErrorMessage then compares it to the UniformInterfaceException + * to make sure that we got what we expected. + * @param expectedStatus + * @param expectedErrorMessage + * @param uie + */ + public void errorParse(int expectedStatus, String expectedErrorMessage, UniformInterfaceException uie){ + assertEquals(expectedStatus,uie.getResponse().getStatus()); + JsonNode errorJson = uie.getResponse().getEntity( JsonNode.class ); + assertEquals( expectedErrorMessage, errorJson.get( "error" ).asText() ); + + } + + + protected Token getAdminToken(String username, String password){ + return this.clientSetup.getRestClient().management().token().post( + new Token(username, password) + ); + } + + protected Token getAdminToken(){ + return this.clientSetup.getRestClient().management().token().post( + new Token(this.clientSetup.getUsername(),this.clientSetup.getUsername()) + ); + } +} http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/a7840164/stack/rest/src/test/java/org/apache/usergrid/rest/test/resource2point0/ClientSetup.java ---------------------------------------------------------------------- diff --cc stack/rest/src/test/java/org/apache/usergrid/rest/test/resource2point0/ClientSetup.java index 3455744,0000000..e033c2d mode 100644,000000..100644 --- a/stack/rest/src/test/java/org/apache/usergrid/rest/test/resource2point0/ClientSetup.java +++ b/stack/rest/src/test/java/org/apache/usergrid/rest/test/resource2point0/ClientSetup.java @@@ -1,140 -1,0 +1,152 @@@ +/** + * Created by ApigeeCorporation on 12/4/14. + */ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.usergrid.rest.test.resource2point0; + + +import java.io.IOException; + +import org.apache.usergrid.rest.test.resource2point0.model.Application; +import org.apache.usergrid.rest.test.resource2point0.model.Token; +import org.junit.rules.TestRule; +import org.junit.runner.Description; +import org.junit.runners.model.Statement; + +import org.apache.usergrid.persistence.index.utils.UUIDUtils; +import org.apache.usergrid.rest.test.resource2point0.model.Organization; ++import org.slf4j.Logger; ++import org.slf4j.LoggerFactory; + +import javax.ws.rs.core.MediaType; + + +/** + * This class is used to setup the client rule that will setup the RestClient and create default applications. + */ +public class ClientSetup implements TestRule { + ++ private Logger logger = LoggerFactory.getLogger( ClientSetup.class ); ++ + RestClient restClient; + + protected String username; + protected String password; + protected String orgName; + protected String appName; + protected Token superuserToken; + protected String superuserName = "superuser"; + protected String superuserPassword = "superpassword"; + + protected Organization organization; + protected Application application; + + + public ClientSetup (String serverUrl) { + + restClient = new RestClient( serverUrl ); + } + + public Statement apply( Statement base, Description description ) { + return statement( base, description ); + } + + + private Statement statement( final Statement base, final Description description ) { + return new Statement() { + @Override + public void evaluate() throws Throwable { + before( description ); + try { + base.evaluate(); + } + finally { + cleanup(); + } + } + }; + } + + + protected void cleanup() { + // might want to do something here later + } + + + protected void before( Description description ) throws IOException { + String testClass = description.getTestClass().getName(); + String methodName = description.getMethodName(); + String name = testClass + "." + methodName; + - restClient.superuserSetup(); - superuserToken = restClient.management().token().post( new Token( superuserName, superuserPassword ) ); ++ try { ++ restClient.superuserSetup(); ++ superuserToken = restClient.management().token().post( new Token( superuserName, superuserPassword ) ); ++ } catch ( Exception e ) { ++ if ( logger.isDebugEnabled() ) { ++ logger.debug( "Error creating superuser, may already exist", e ); ++ } else { ++ logger.warn( "Error creating superuser, may already exist"); ++ } ++ } + + username = "user_"+name + UUIDUtils.newTimeUUID(); + password = username; + orgName = "org_"+name+UUIDUtils.newTimeUUID(); + appName = "app_"+name+UUIDUtils.newTimeUUID(); + + organization = restClient.management().orgs().post(new Organization( orgName,username,username+"@usergrid.com",username,username, null )); + + restClient.management().token().post(new Token(username,username)); + + restClient.management().orgs().organization(organization.getName()).app().post(new Application(appName)); + + } + + public String getUsername(){return username;} + + public String getEmail(){return username+"@usergrid.com";} + + public String getPassword(){return password;} + + public Organization getOrganization(){return organization;} + + public String getOrganizationName(){return orgName;} + + public String getAppName() {return appName;} + + public Token getSuperuserToken() { + return superuserToken; + } + + public String getSuperuserName() { + return superuserName; + } + + public String getSuperuserPassword() { + return superuserPassword; + } + + public void refreshIndex() { + this.restClient.refreshIndex(getOrganizationName(),getAppName()); + } + + public RestClient getRestClient(){ + return restClient; + } +} http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/a7840164/stack/services/pom.xml ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/a7840164/stack/services/src/main/java/org/apache/usergrid/management/ManagementService.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/a7840164/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java ---------------------------------------------------------------------- diff --cc stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java index 0c319ca,5b77534..85ddcbb --- a/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java +++ b/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java @@@ -1443,9 -1391,16 +1443,16 @@@ public class ManagementServiceImpl impl } + @Override + public void importTokenForAdminUser(UUID userId, String token, long ttl) throws Exception { + tokens.importToken( token, TokenCategory.ACCESS, null, - new AuthPrincipalInfo( ADMIN_USER, userId, MANAGEMENT_APPLICATION_ID ), null, ttl ); ++ new AuthPrincipalInfo( ADMIN_USER, userId, smf.getManagementAppId() ), null, ttl ); + } + + /* * (non-Javadoc) - * + * * @see * org.apache.usergrid.management.ManagementService#revokeAccessTokensForAdminUser * (java.util.UUID) http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/a7840164/stack/services/src/main/java/org/apache/usergrid/security/shiro/Realm.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/a7840164/stack/services/src/main/java/org/apache/usergrid/security/tokens/cassandra/TokenServiceImpl.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/a7840164/stack/services/src/main/java/org/apache/usergrid/services/assets/data/S3BinaryStore.java ---------------------------------------------------------------------- diff --cc stack/services/src/main/java/org/apache/usergrid/services/assets/data/S3BinaryStore.java index 90bd24f,f59e79c..a805e42 --- a/stack/services/src/main/java/org/apache/usergrid/services/assets/data/S3BinaryStore.java +++ b/stack/services/src/main/java/org/apache/usergrid/services/assets/data/S3BinaryStore.java @@@ -17,49 -17,45 +17,38 @@@ package org.apache.usergrid.services.assets.data; --import java.io.BufferedOutputStream; --import java.io.ByteArrayOutputStream; --import java.io.File; --import java.io.FileOutputStream; --import java.io.IOException; --import java.io.InputStream; --import java.io.OutputStream; - import java.security.MessageDigest; - import java.security.NoSuchAlgorithmException; --import java.util.Map; -import java.util.Properties; -import java.util.Stack; --import java.util.UUID; - import java.util.concurrent.ExecutorService; - import java.util.concurrent.Executors; -import java.util.concurrent.*; -- - import com.google.common.hash.HashCode; ++import com.google.common.collect.ImmutableSet; +import com.google.common.hash.HashFunction; +import com.google.common.hash.Hashing; +import com.google.common.io.Files; ++import com.google.inject.Module; ++import org.apache.commons.codec.binary.Hex; ++import org.apache.commons.io.FileUtils; ++import org.apache.commons.io.IOUtils; ++import org.apache.usergrid.persistence.Entity; ++import org.apache.usergrid.persistence.EntityManager; ++import org.apache.usergrid.persistence.EntityManagerFactory; + import org.apache.usergrid.utils.StringUtils; import org.jclouds.ContextBuilder; --import org.jclouds.blobstore.AsyncBlobStore; import org.jclouds.blobstore.BlobStore; import org.jclouds.blobstore.BlobStoreContext; import org.jclouds.blobstore.domain.Blob; import org.jclouds.blobstore.domain.BlobBuilder; import org.jclouds.blobstore.options.GetOptions; --import org.jclouds.blobstore.options.PutOptions; import org.jclouds.http.config.JavaUrlHttpCommandExecutorServiceModule; import org.jclouds.logging.log4j.config.Log4JLoggingModule; import org.jclouds.netty.config.NettyPayloadModule; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; --import org.apache.usergrid.persistence.Entity; --import org.apache.usergrid.persistence.EntityManager; --import org.apache.usergrid.persistence.EntityManagerFactory; - - import org.apache.commons.codec.binary.Hex; - import org.apache.commons.io.FileUtils; - import org.apache.commons.io.IOUtils; -import org.apache.commons.codec.binary.Hex; -import org.apache.commons.io.FileUtils; -import org.apache.commons.io.IOUtils; - --import com.google.common.collect.ImmutableSet; --import com.google.common.util.concurrent.ListenableFuture; --import com.google.inject.Module; ++import java.io.*; ++import java.util.Map; ++import java.util.Properties; ++import java.util.UUID; ++import java.util.concurrent.Callable; ++import java.util.concurrent.ExecutorService; ++import java.util.concurrent.Executors; public class S3BinaryStore implements BinaryStore { @@@ -115,18 -116,17 +109,19 @@@ long written = IOUtils.copyLarge( inputStream, baos, 0, FIVE_MB ); byte[] data = baos.toByteArray(); - final Map<String, Object> fileMetadata = AssetUtils.getFileMetadata( entity ); - fileMetadata.put( AssetUtils.LAST_MODIFIED, System.currentTimeMillis() ); + if ( written < FIVE_MB ) { // total smaller than 5mb - String mimeType = AssetMimeHandler.get().getMimeType( entity, data ); + final String uploadFileName = AssetUtils.buildAssetKey( appId, entity ); + final String mimeType = AssetMimeHandler.get().getMimeType( entity, data ); - if ( written < FIVE_MB ) { // total smaller than 5mb + final Map<String, Object> fileMetadata = AssetUtils.getFileMetadata( entity ); + fileMetadata.put( AssetUtils.LAST_MODIFIED, System.currentTimeMillis() ); BlobStore blobStore = getContext().getBlobStore(); - BlobBuilder.PayloadBlobBuilder bb = blobStore.blobBuilder( uploadFileName ) - .payload( data ).calculateMD5().contentType( mimeType ); + BlobBuilder.PayloadBlobBuilder bb = blobStore.blobBuilder(uploadFileName) - .payload(data) - .contentMD5(Hashing.md5().newHasher().putBytes( data ).hash()) - .contentType(mimeType); ++ .payload( data ) ++ .contentMD5( Hashing.md5().newHasher().putBytes( data ).hash() ) ++ .contentType( mimeType ); fileMetadata.put( AssetUtils.CONTENT_LENGTH, written ); if ( fileMetadata.get( AssetUtils.CONTENT_DISPOSITION ) != null ) { @@@ -142,56 -142,31 +137,30 @@@ } else { // bigger than 5mb... dump 5 mb tmp files and upload from them - // todo: yes, AsyncBlobStore is deprecated, but there appears to be no replacement yet - final AsyncBlobStore blobStore = getContext().getAsyncBlobStore(); + ExecutorService executors = getExecutorService(); - File tempFile = File.createTempFile( entity.getUuid().toString(), "tmp" ); - tempFile.deleteOnExit(); - OutputStream os = null; - try { - os = new BufferedOutputStream( new FileOutputStream( tempFile.getAbsolutePath() ) ); - os.write( data ); - written += IOUtils.copyLarge( inputStream, os, 0, ( FileUtils.ONE_GB * 5 ) ); - } - finally { - IOUtils.closeQuietly( os ); - } + executors.submit( new UploadWorker( appId, entity, inputStream, data, written ) ); + } + } - BlobBuilder.PayloadBlobBuilder bb = blobStore.blobBuilder( uploadFileName ) - .payload(tempFile) - .contentMD5(Files.hash(tempFile, Hashing.md5())) - .contentType(mimeType); - + private ExecutorService getExecutorService() { - fileMetadata.put( AssetUtils.CONTENT_LENGTH, written ); - if ( fileMetadata.get( AssetUtils.CONTENT_DISPOSITION ) != null ) { - bb.contentDisposition( fileMetadata.get( AssetUtils.CONTENT_DISPOSITION ).toString() ); - } - final Blob blob = bb.build(); + if ( executorService == null ) { + synchronized (this) { - final File finalTempFile = tempFile; - final ListenableFuture<String> future = - blobStore.putBlob( bucketName, blob, PutOptions.Builder.multipart() ); + int workers = 40; + String workersString = properties.getProperty( WORKERS_PROP_NAME, "40"); - Runnable listener = new Runnable() { - @Override - public void run() { - try { - String eTag = future.get(); - fileMetadata.put( AssetUtils.E_TAG, eTag ); - EntityManager em = emf.getEntityManager( appId ); - em.update( entity ); - finalTempFile.delete(); - } - catch ( Exception e ) { - LOG.error( "error uploading", e ); - } - if ( finalTempFile != null && finalTempFile.exists() ) { - finalTempFile.delete(); - } + if ( StringUtils.isNumeric( workersString ) ) { + workers = Integer.parseInt( workersString ); + } else if ( !StringUtils.isEmpty( workersString )) { + LOG.error("Ignoring invalid setting for {}", WORKERS_PROP_NAME); } - }; - future.addListener( listener, executor ); + executorService = Executors.newFixedThreadPool( workers ); + } } + + return executorService; } @@@ -224,5 -199,124 +193,133 @@@ BlobStore blobStore = getContext().getBlobStore(); blobStore.removeBlob( bucketName, AssetUtils.buildAssetKey( appId, entity ) ); } + + class UploadWorker implements Callable<Void> { + + private UUID appId; + private Entity entity; + private InputStream inputStream; + private byte[] data; + private long written; + + + public UploadWorker( UUID appId, Entity entity, InputStream is, byte[] data, long written ) { + this.appId = appId; + this.entity = entity; + this.inputStream = is; + this.data = data; + this.written = written; + } + + @Override + public Void call() { + + LOG.debug( "Writing temp file for S3 upload" ); + + // determine max size file allowed, default to 50mb + long maxSizeBytes = 50 * FileUtils.ONE_MB; + String maxSizeMbString = properties.getProperty( "usergrid.binary.max-size-mb", "50" ); + if (StringUtils.isNumeric( maxSizeMbString )) { + maxSizeBytes = Long.parseLong( maxSizeMbString ) * FileUtils.ONE_MB; + } + + // always allow files up to 5mb + if (maxSizeBytes < 5 * FileUtils.ONE_MB ) { + maxSizeBytes = 5 * FileUtils.ONE_MB; + } + + // write temporary file, slightly larger than our size limit + OutputStream os = null; + File tempFile; + try { + tempFile = File.createTempFile( entity.getUuid().toString(), "tmp" ); + tempFile.deleteOnExit(); + os = new BufferedOutputStream( new FileOutputStream( tempFile.getAbsolutePath() ) ); + os.write( data ); ++ written += data.length; + written += IOUtils.copyLarge( inputStream, os, 0, maxSizeBytes + 1 ); + ++ LOG.debug("Write temp file {} length {}", tempFile.getName(), written); ++ + } catch ( IOException e ) { + throw new RuntimeException( "Error creating temp file", e ); + + } finally { + if ( os != null ) { ++ try { ++ os.flush(); ++ } catch (IOException e) { ++ LOG.error( "Error flushing data to temporary upload file", e ); ++ } + IOUtils.closeQuietly( os ); + } + } + + // if tempFile is too large, delete it, add error to entity file metadata and abort + + Map<String, Object> fileMetadata = AssetUtils.getFileMetadata( entity ); + + if ( tempFile.length() > maxSizeBytes ) { + LOG.debug("File too large. Temp file size (bytes) = {}, " + - "Max file size (bytes) = {} ", tempFile.length(), maxSizeBytes); ++ "Max file size (bytes) = {} ", tempFile.length(), maxSizeBytes); + try { + EntityManager em = emf.getEntityManager( appId ); + fileMetadata.put( "error", "Asset size " + tempFile.length() + + " is larger than max size of " + maxSizeBytes ); + em.update( entity ); + tempFile.delete(); + + } catch ( Exception e ) { + LOG.error( "Error updating entity with error message", e); + } + return null; + } + + String uploadFileName = AssetUtils.buildAssetKey( appId, entity ); + String mimeType = AssetMimeHandler.get().getMimeType( entity, data ); + + try { // start the upload + + LOG.debug( "S3 upload thread started" ); + + BlobStore blobStore = getContext().getBlobStore(); + - BlobBuilder.PayloadBlobBuilder bb = blobStore.blobBuilder( uploadFileName ) - .payload( tempFile ).calculateMD5().contentType( mimeType ); ++ BlobBuilder.PayloadBlobBuilder bb = blobStore.blobBuilder( uploadFileName ) ++ .payload( tempFile ) ++ .contentMD5( Files.hash( tempFile, Hashing.md5() ) ) ++ .contentType( mimeType ); + + if ( fileMetadata.get( AssetUtils.CONTENT_DISPOSITION ) != null ) { + bb.contentDisposition( fileMetadata.get( AssetUtils.CONTENT_DISPOSITION ).toString() ); + } + final Blob blob = bb.build(); + + String md5sum = Hex.encodeHexString( blob.getMetadata().getContentMetadata().getContentMD5() ); + fileMetadata.put( AssetUtils.CHECKSUM, md5sum ); + + LOG.debug( "S3 upload starting" ); + + String eTag = blobStore.putBlob( bucketName, blob ); + + LOG.debug( "S3 upload complete eTag=" + eTag); + - // update entity with information about uploaded asset - ++ // update entity with eTag + EntityManager em = emf.getEntityManager( appId ); - fileMetadata.put( AssetUtils.E_TAG, eTag ); + fileMetadata.put( AssetUtils.LAST_MODIFIED, System.currentTimeMillis() ); + fileMetadata.put( AssetUtils.CONTENT_LENGTH, written ); ++ fileMetadata.put( AssetUtils.E_TAG, eTag ); + em.update( entity ); + } + catch ( Exception e ) { + LOG.error( "error uploading", e ); + } + + if ( tempFile != null && tempFile.exists() ) { + tempFile.delete(); + } + + return null; + } + } } http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/a7840164/stack/services/src/test/java/org/apache/usergrid/security/tokens/TokenServiceIT.java ----------------------------------------------------------------------
