Repository: usergrid Updated Branches: refs/heads/master 2867d26d6 -> 41caeaa10
Tools now able to compile again. Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/b5144d05 Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/b5144d05 Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/b5144d05 Branch: refs/heads/master Commit: b5144d058189c37a639fa29d67056c0c96ff8531 Parents: fedf165 Author: Dave Johnson <[email protected]> Authored: Wed Jul 22 11:16:33 2015 -0400 Committer: Dave Johnson <[email protected]> Committed: Wed Jul 22 11:16:33 2015 -0400 ---------------------------------------------------------------------- .../usergrid/persistence/EntityManager.java | 4 +- stack/pom.xml | 2 +- .../usergrid/services/AbstractService.java | 270 ++++++++++--------- stack/tools/pom.xml | 4 +- .../java/org/apache/usergrid/tools/Cli.java | 35 +-- .../org/apache/usergrid/tools/DupOrgRepair.java | 2 +- .../org/apache/usergrid/tools/ExportAdmins.java | 34 +-- .../org/apache/usergrid/tools/ExportApp.java | 92 +++---- .../usergrid/tools/ExportDataCreator.java | 7 +- .../org/apache/usergrid/tools/ImportAdmins.java | 104 +++---- .../java/org/apache/usergrid/tools/Metrics.java | 2 +- .../usergrid/tools/OrganizationExport.java | 2 +- .../apache/usergrid/tools/PopulateSample.java | 2 +- .../org/apache/usergrid/tools/UserManager.java | 3 +- .../apache/usergrid/tools/WarehouseExport.java | 117 +++----- .../apache/usergrid/tools/bean/MetricQuery.java | 6 +- .../apache/usergrid/tools/ExportAppTest.java | 39 +-- .../usergrid/tools/ExportImportAdminsTest.java | 26 +- 18 files changed, 346 insertions(+), 405 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/usergrid/blob/b5144d05/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java ---------------------------------------------------------------------- diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java b/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java index c4a77ac..c544914 100644 --- a/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java +++ b/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java @@ -463,14 +463,14 @@ public interface EntityManager { * * @param entityRef an entity reference * @param connectionType type of connection or null. - * @param connectedEntityType type of entity or null. + * @param targetEntityType type of entity or null. * * @return a list of connected entity ids. * * @throws Exception the exception */ public Results getTargetEntities(EntityRef entityRef, String connectionType, - String connectedEntityType, Level resultsLevel) throws Exception; + String targetEntityType, Level resultsLevel) throws Exception; /** * Gets the entities connecting to this entity, optionally with the specified connection http://git-wip-us.apache.org/repos/asf/usergrid/blob/b5144d05/stack/pom.xml ---------------------------------------------------------------------- diff --git a/stack/pom.xml b/stack/pom.xml index 6f984e3..c03e52f 100644 --- a/stack/pom.xml +++ b/stack/pom.xml @@ -190,8 +190,8 @@ <module>core</module> <module>services</module> <module>rest</module> - <!-- <module>tools</module> + <!-- <module>websocket</module> --> <!-- http://git-wip-us.apache.org/repos/asf/usergrid/blob/b5144d05/stack/services/src/main/java/org/apache/usergrid/services/AbstractService.java ---------------------------------------------------------------------- diff --git a/stack/services/src/main/java/org/apache/usergrid/services/AbstractService.java b/stack/services/src/main/java/org/apache/usergrid/services/AbstractService.java index edf8ab2..1c04da9 100644 --- a/stack/services/src/main/java/org/apache/usergrid/services/AbstractService.java +++ b/stack/services/src/main/java/org/apache/usergrid/services/AbstractService.java @@ -26,6 +26,8 @@ import java.util.Map; import java.util.Set; import java.util.UUID; +import com.codahale.metrics.Timer; +import org.apache.usergrid.persistence.core.metrics.MetricsFactory; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.context.ApplicationContext; @@ -98,8 +100,11 @@ public abstract class AbstractService implements Service { private Scheduler rxScheduler; private RxSchedulerFig rxSchedulerFig; - - + private MetricsFactory metricsFactory; + private Timer entityGetTimer; + private Timer entitiesGetTimer; + private Timer entitiesParallelGetTimer; + private Timer invokeTimer; public AbstractService() { @@ -112,7 +117,12 @@ public abstract class AbstractService implements Service { em = sm.getEntityManager(); final Injector injector = sm.getApplicationContext().getBean( Injector.class ); rxScheduler = injector.getInstance( RxTaskScheduler.class ).getAsyncIOScheduler(); - rxSchedulerFig = injector.getInstance( RxSchedulerFig.class ); + rxSchedulerFig = injector.getInstance(RxSchedulerFig.class); + metricsFactory = injector.getInstance(MetricsFactory.class); + this.entityGetTimer = metricsFactory.getTimer(this.getClass(), "importEntity.get"); + this.entitiesGetTimer = metricsFactory.getTimer(this.getClass(), "importEntities.get"); + this.entitiesParallelGetTimer = metricsFactory.getTimer( this.getClass(),"importEntitiesP.get" ); + this.invokeTimer = metricsFactory.getTimer( this.getClass(),"service.invoke" ); } @@ -170,9 +180,9 @@ public abstract class AbstractService implements Service { return false; } if ( "application".equals( context.getOwner().getType() ) ) { - return Schema.getDefaultSchema().isCollectionReversed( "application", pluralize( info.getItemType() ) ); + return Schema.getDefaultSchema().isCollectionReversed("application", pluralize(info.getItemType())); } - return Schema.getDefaultSchema().isCollectionReversed( info.getContainerType(), info.getCollectionName() ); + return Schema.getDefaultSchema().isCollectionReversed(info.getContainerType(), info.getCollectionName()); } @@ -181,9 +191,9 @@ public abstract class AbstractService implements Service { return null; } if ( "application".equals( context.getOwner().getType() ) ) { - return Schema.getDefaultSchema().getCollectionSort( "application", pluralize( info.getItemType() ) ); + return Schema.getDefaultSchema().getCollectionSort("application", pluralize(info.getItemType())); } - return Schema.getDefaultSchema().getCollectionSort( info.getContainerType(), info.getCollectionName() ); + return Schema.getDefaultSchema().getCollectionSort(info.getContainerType(), info.getCollectionName()); } @@ -191,7 +201,7 @@ public abstract class AbstractService implements Service { if ( privateConnections == null ) { privateConnections = new LinkedHashSet<String>(); } - privateConnections.add( connection ); + privateConnections.add(connection); } @@ -199,7 +209,7 @@ public abstract class AbstractService implements Service { if ( privateConnections == null ) { privateConnections = new LinkedHashSet<String>(); } - privateConnections.addAll( connections ); + privateConnections.addAll(connections); } @@ -207,7 +217,7 @@ public abstract class AbstractService implements Service { if ( declaredConnections == null ) { declaredConnections = new LinkedHashSet<String>(); } - declaredConnections.add( connection ); + declaredConnections.add(connection); } @@ -215,7 +225,7 @@ public abstract class AbstractService implements Service { if ( declaredConnections == null ) { declaredConnections = new LinkedHashSet<String>(); } - declaredConnections.addAll( connections ); + declaredConnections.addAll(connections); } @@ -223,7 +233,7 @@ public abstract class AbstractService implements Service { if ( privateCollections == null ) { privateCollections = new LinkedHashSet<String>(); } - privateCollections.add( collection ); + privateCollections.add(collection); } @@ -231,7 +241,7 @@ public abstract class AbstractService implements Service { if ( privateCollections == null ) { privateCollections = new LinkedHashSet<String>(); } - privateCollections.addAll( collections ); + privateCollections.addAll(collections); } @@ -239,7 +249,7 @@ public abstract class AbstractService implements Service { if ( declaredCollections == null ) { declaredCollections = new LinkedHashSet<String>(); } - declaredCollections.add( collection ); + declaredCollections.add(collection); } @@ -247,7 +257,7 @@ public abstract class AbstractService implements Service { if ( declaredCollections == null ) { declaredCollections = new LinkedHashSet<String>(); } - declaredCollections.addAll( collections ); + declaredCollections.addAll(collections); } @@ -255,7 +265,7 @@ public abstract class AbstractService implements Service { if ( replaceParameters == null ) { replaceParameters = new LinkedHashMap<List<String>, List<String>>(); } - replaceParameters.put( find, replace ); + replaceParameters.put(find, replace); } @@ -263,7 +273,7 @@ public abstract class AbstractService implements Service { if ( serviceCommands == null ) { serviceCommands = new LinkedHashSet<String>(); } - serviceCommands.add( command ); + serviceCommands.add(command); } @@ -271,7 +281,7 @@ public abstract class AbstractService implements Service { if ( serviceCommands == null ) { serviceCommands = new LinkedHashSet<String>(); } - serviceCommands.addAll( commands ); + serviceCommands.addAll(commands); } @@ -279,7 +289,7 @@ public abstract class AbstractService implements Service { if ( entityDictionaries == null ) { entityDictionaries = new LinkedHashSet<EntityDictionaryEntry>(); } - entityDictionaries.add( dictionary ); + entityDictionaries.add(dictionary); } @@ -305,7 +315,7 @@ public abstract class AbstractService implements Service { if ( metadataTypes == null ) { metadataTypes = new LinkedHashSet<String>(); } - metadataTypes.add( type ); + metadataTypes.add(type); } @@ -313,7 +323,7 @@ public abstract class AbstractService implements Service { if ( metadataTypes == null ) { metadataTypes = new LinkedHashSet<String>(); } - metadataTypes.addAll( typeList ); + metadataTypes.addAll(typeList); } @@ -321,7 +331,7 @@ public abstract class AbstractService implements Service { if ( entityCommands == null ) { entityCommands = new LinkedHashSet<String>(); } - entityCommands.add( command ); + entityCommands.add(command); } @@ -329,7 +339,7 @@ public abstract class AbstractService implements Service { if ( entityCommands == null ) { entityCommands = new LinkedHashSet<String>(); } - entityCommands.addAll( commands ); + entityCommands.addAll(commands); } @@ -354,69 +364,79 @@ public abstract class AbstractService implements Service { @Override public Entity importEntity( ServiceRequest request, Entity entity ) throws Exception { - if ( entity == null ) { - return null; - } + Timer.Context getEntityTimer = entityGetTimer.time(); + try { + if (entity == null) { + return null; + } - if ( !isRootService() ) { - return sm.importEntity( request, entity ); - } + if (!isRootService()) { + return sm.importEntity(request, entity); + } - String path = request.getPath() + "/" + entity.getUuid(); - Map<String, Object> metadata = new LinkedHashMap<String, Object>(); - metadata.put( "path", path ); - if ( defaultEntityMetadata != null ) { - metadata.putAll( defaultEntityMetadata ); - } + String path = request.getPath() + "/" + entity.getUuid(); + Map<String, Object> metadata = new LinkedHashMap<String, Object>(); + metadata.put("path", path); - Set<Object> connections = getConnectedTypesSet( entity ); - if ( connections != null ) { - Map<String, Object> m = new LinkedHashMap<String, Object>(); - for ( Object n : connections ) { - m.put( n.toString(), path + "/" + n ); + if (defaultEntityMetadata != null) { + metadata.putAll(defaultEntityMetadata); + } + + Set<Object> connections = getConnectedTypesSet(entity); + if (connections != null) { + Map<String, Object> m = new LinkedHashMap<String, Object>(); + for (Object n : connections) { + m.put(n.toString(), path + "/" + n); + } + metadata.put("connections", m); } - metadata.put( "connections", m ); - } - Set<Object> connecting = getConnectingTypesSet( entity ); - if ( connecting != null ) { - Map<String, Object> m = new LinkedHashMap<String, Object>(); - for ( Object n : connecting ) { - m.put( n.toString(), path + "/connecting/" + n ); + Set<Object> connecting = getConnectingTypesSet(entity); + if (connecting != null) { + Map<String, Object> m = new LinkedHashMap<String, Object>(); + for (Object n : connecting) { + m.put(n.toString(), path + "/connecting/" + n); + } + metadata.put("connecting", m); } - metadata.put( "connecting", m ); - } - Set<String> collections = getCollectionSet( entity ); - if ( collections != null ) { - Map<String, Object> m = new LinkedHashMap<String, Object>(); - for ( Object n : collections ) { - m.put( n.toString(), path + "/" + n ); + Set<String> collections = getCollectionSet(entity); + if (collections != null) { + Map<String, Object> m = new LinkedHashMap<String, Object>(); + for (Object n : collections) { + m.put(n.toString(), path + "/" + n); + } + metadata.put("collections", m); } - metadata.put( "collections", m ); - } - if ( entityDictionaries != null ) { - Map<String, Object> m = new LinkedHashMap<String, Object>(); - for ( EntityDictionaryEntry dict : entityDictionaries ) { - m.put( dict.getName(), path + "/" + dict.getPath() ); + if (entityDictionaries != null) { + Map<String, Object> m = new LinkedHashMap<String, Object>(); + for (EntityDictionaryEntry dict : entityDictionaries) { + m.put(dict.getName(), path + "/" + dict.getPath()); + } + metadata.put("sets", m); } - metadata.put( "sets", m ); - } - if ( metadata.size() > 0 ) { - entity.mergeMetadata( metadata ); + if (metadata.size() > 0) { + entity.mergeMetadata(metadata); + } + return entity; + }finally { + getEntityTimer.stop(); } - return entity; } public void importEntities( ServiceRequest request, Results results ) throws Exception { - - List<Entity> entities = results.getEntities(); - if ( entities != null ) { - importEntitiesParallel(request, results); + Timer.Context timer = entitiesGetTimer.time(); + try { + List<Entity> entities = results.getEntities(); + if (entities != null) { + importEntitiesParallel(request, results); + } + }finally { + timer.stop(); } } @@ -427,42 +447,45 @@ public abstract class AbstractService implements Service { * @param results */ private void importEntitiesParallel(final ServiceRequest request, final Results results ) { + Timer.Context timer = entitiesParallelGetTimer.time(); + try { + //create our tuples + final Observable<EntityTuple> tuples = Observable.create(new Observable.OnSubscribe<EntityTuple>() { + @Override + public void call(final Subscriber<? super EntityTuple> subscriber) { + subscriber.onStart(); + + final List<Entity> entities = results.getEntities(); + final int size = entities.size(); + for (int i = 0; i < size && !subscriber.isUnsubscribed(); i++) { + subscriber.onNext(new EntityTuple(i, entities.get(i))); + } - //create our tuples - final Observable<EntityTuple> tuples = Observable.create( new Observable.OnSubscribe<EntityTuple>() { - @Override - public void call( final Subscriber<? super EntityTuple> subscriber ) { - subscriber.onStart(); - - final List<Entity> entities = results.getEntities(); - final int size = entities.size(); - for ( int i = 0; i < size && !subscriber.isUnsubscribed(); i++ ) { - subscriber.onNext( new EntityTuple( i, entities.get( i ) ) ); + subscriber.onCompleted(); } + }); - subscriber.onCompleted(); - } - } ); - - //now process them in parallel up to 10 threads + //now process them in parallel up to 10 threads - tuples.flatMap( tuple -> { - //map the entity into the tuple - return Observable.just( tuple ).doOnNext( parallelTuple -> { - //import the entity and set it at index - try { + tuples.flatMap(tuple -> { + //map the entity into the tuple + return Observable.just(tuple).doOnNext(parallelTuple -> { + //import the entity and set it at index + try { - final Entity imported = importEntity( request, parallelTuple.entity ); + final Entity imported = importEntity(request, parallelTuple.entity); - if(imported != null) { - results.setEntity( parallelTuple.index, imported ); + if (imported != null) { + results.setEntity(parallelTuple.index, imported); + } + } catch (Exception e) { + throw new RuntimeException(e); } - } - catch ( Exception e ) { - throw new RuntimeException(e); - } - } ).subscribeOn( rxScheduler ); - }, rxSchedulerFig.getImportThreads() ).toBlocking().lastOrDefault( null ); + }).subscribeOn(rxScheduler); + }, rxSchedulerFig.getImportThreads()).toBlocking().lastOrDefault(null); + } finally { + timer.stop(); + } } @@ -528,7 +551,7 @@ public abstract class AbstractService implements Service { entity.addProperties( payload.getProperties() ); return entity; } - logger.error( "Attempted update of entity reference rather than full entity, currently unsupport - MUSTFIX" ); + logger.error("Attempted update of entity reference rather than full entity, currently unsupport - MUSTFIX"); throw new NotImplementedException(); } @@ -555,7 +578,7 @@ public abstract class AbstractService implements Service { public Set<Object> getConnectedTypesSet( EntityRef ref ) throws Exception { - final Set<String> connections = em.getConnectionsAsSource( ref ); + final Set<String> connections = em.getConnectionsAsSource(ref); if ( connections == null ) { return null; @@ -574,7 +597,7 @@ public abstract class AbstractService implements Service { public Set<Object> getConnectingTypesSet( EntityRef ref ) throws Exception { - final Set<String> connections = em.getConnectionsAsTarget( ref ); + final Set<String> connections = em.getConnectionsAsTarget(ref); if ( connections == null ) { return null; @@ -593,7 +616,7 @@ public abstract class AbstractService implements Service { public Set<String> getCollectionSet( EntityRef ref ) { - Set<String> set = Schema.getDefaultSchema().getCollectionNames( ref.getType() ); + Set<String> set = Schema.getDefaultSchema().getCollectionNames(ref.getType()); set = new LinkedHashSet<String>( set ); if ( declaredCollections != null ) { set.addAll( declaredCollections ); @@ -612,7 +635,7 @@ public abstract class AbstractService implements Service { public ServiceResults invoke( ServiceAction action, ServiceRequest request, ServiceResults previousResults, ServicePayload payload ) throws Exception { - ServiceContext context = getContext( action, request, previousResults, payload ); + ServiceContext context = getContext(action, request, previousResults, payload); return invoke( context ); } @@ -629,7 +652,7 @@ public abstract class AbstractService implements Service { EntityRef owner = request.getOwner(); String collectionName = "application".equals( owner.getType() ) ? pluralize( info.getItemType() ) : info.getCollectionName(); - List<ServiceParameter> parameters = filter( request.getParameters(), replaceParameters ); + List<ServiceParameter> parameters = filter(request.getParameters(), replaceParameters); ServiceParameter first_parameter = null; if ( !isEmpty( parameters ) ) { @@ -652,7 +675,7 @@ public abstract class AbstractService implements Service { if ( first_parameter instanceof QueryParameter ) { query = first_parameter.getQuery(); } - parameters = mergeQueries( query, parameters ); + parameters = mergeQueries(query, parameters); if ( first_parameter instanceof IdParameter ) { UUID id = first_parameter.getId(); @@ -678,9 +701,8 @@ public abstract class AbstractService implements Service { public ServiceResults invoke( ServiceContext context ) throws Exception { - ServiceResults results = null; - + Timer.Context time = invokeTimer.time(); String metadataType = checkForServiceMetadata( context ); if ( metadataType != null ) { return handleServiceMetadata( context, metadataType ); @@ -708,8 +730,9 @@ public abstract class AbstractService implements Service { } results = handleEntityDictionary( context, results, entityDictionary ); - results = handleEntityCommand( context, results, entityCommand ); + results = handleEntityCommand(context, results, entityCommand); + time.stop(); return results; } @@ -718,19 +741,19 @@ public abstract class AbstractService implements Service { switch ( context.getAction() ) { case GET: - return getItemById( context, id ); + return getItemById(context, id); case POST: - return postItemById( context, id ); + return postItemById(context, id); case PUT: - return putItemById( context, id ); + return putItemById(context, id); case DELETE: - return deleteItemById( context, id ); + return deleteItemById(context, id); case HEAD: - return headItemById( context, id ); + return headItemById(context, id); } throw new ServiceInvocationException( context, "Request action unhandled " + context.getAction() ); @@ -738,25 +761,20 @@ public abstract class AbstractService implements Service { public ServiceResults invokeItemWithName( ServiceContext context, String name ) throws Exception { - - switch ( context.getAction() ) { + switch (context.getAction()) { case GET: - return getItemByName( context, name ); - + return getItemByName(context, name); case POST: - return postItemByName( context, name ); - + return postItemByName(context, name); case PUT: - return putItemByName( context, name ); - + return putItemByName(context, name); case DELETE: - return deleteItemByName( context, name ); - + return deleteItemByName(context, name); case HEAD: - return headItemByName( context, name ); + return headItemByName(context, name); + default: + throw new ServiceInvocationException(context, "Request action unhandled " + context.getAction()); } - - throw new ServiceInvocationException( context, "Request action unhandled " + context.getAction() ); } http://git-wip-us.apache.org/repos/asf/usergrid/blob/b5144d05/stack/tools/pom.xml ---------------------------------------------------------------------- diff --git a/stack/tools/pom.xml b/stack/tools/pom.xml index d7bfc98..6e0dc50 100644 --- a/stack/tools/pom.xml +++ b/stack/tools/pom.xml @@ -21,7 +21,7 @@ <parent> <groupId>org.apache.usergrid</groupId> <artifactId>usergrid</artifactId> - <version>2.0.0-SNAPSHOT</version> + <version>2.1.0-SNAPSHOT</version> <relativePath>../</relativePath> </parent> @@ -29,7 +29,7 @@ <name>Usergrid Tools</name> <description>Command line tools for Usergrid system.</description> <packaging>jar</packaging> - <version>2.0.0-SNAPSHOT</version> + <version>2.1.0-SNAPSHOT</version> <reporting> <plugins> http://git-wip-us.apache.org/repos/asf/usergrid/blob/b5144d05/stack/tools/src/main/java/org/apache/usergrid/tools/Cli.java ---------------------------------------------------------------------- diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/Cli.java b/stack/tools/src/main/java/org/apache/usergrid/tools/Cli.java index 9594403..511dcaa 100644 --- a/stack/tools/src/main/java/org/apache/usergrid/tools/Cli.java +++ b/stack/tools/src/main/java/org/apache/usergrid/tools/Cli.java @@ -17,6 +17,19 @@ package org.apache.usergrid.tools; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.OptionBuilder; +import org.apache.commons.cli.Options; +import org.apache.usergrid.persistence.Query; +import org.apache.usergrid.services.*; +import org.apache.usergrid.utils.HttpUtils; +import org.apache.usergrid.utils.JsonUtils; +import org.apache.usergrid.utils.UUIDUtils; +import org.codehaus.jackson.JsonFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import java.io.BufferedReader; import java.io.IOException; import java.io.InputStreamReader; @@ -25,26 +38,6 @@ import java.util.List; import java.util.Map; import java.util.UUID; -import org.codehaus.jackson.JsonFactory; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.apache.usergrid.persistence.index.query.Query; -import org.apache.usergrid.services.ServiceAction; -import org.apache.usergrid.services.ServiceManager; -import org.apache.usergrid.services.ServiceParameter; -import org.apache.usergrid.services.ServicePayload; -import org.apache.usergrid.services.ServiceRequest; -import org.apache.usergrid.services.ServiceResults; -import org.apache.usergrid.utils.HttpUtils; -import org.apache.usergrid.utils.JsonUtils; -import org.apache.usergrid.utils.UUIDUtils; - -import org.apache.commons.cli.CommandLine; -import org.apache.commons.cli.Option; -import org.apache.commons.cli.OptionBuilder; -import org.apache.commons.cli.Options; -import org.apache.usergrid.persistence.index.exceptions.QueryParseException; - public class Cli extends ToolBase { @@ -79,7 +72,7 @@ public class Cli extends ToolBase { } - public void handleInput() throws QueryParseException { + public void handleInput() throws Exception { BufferedReader d = new BufferedReader( new InputStreamReader( System.in ) ); UUID applicationId = null; http://git-wip-us.apache.org/repos/asf/usergrid/blob/b5144d05/stack/tools/src/main/java/org/apache/usergrid/tools/DupOrgRepair.java ---------------------------------------------------------------------- diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/DupOrgRepair.java b/stack/tools/src/main/java/org/apache/usergrid/tools/DupOrgRepair.java index a20dda1..ed4deee 100644 --- a/stack/tools/src/main/java/org/apache/usergrid/tools/DupOrgRepair.java +++ b/stack/tools/src/main/java/org/apache/usergrid/tools/DupOrgRepair.java @@ -238,7 +238,7 @@ public class DupOrgRepair extends ExportingToolBase { app.getKey(), app.getValue(), targetOrgId } ); - managementService.addApplicationToOrganization( targetOrgId, app.getValue(), appEntity); + managementService.addApplicationToOrganization( targetOrgId, appEntity); } // now delete the original org http://git-wip-us.apache.org/repos/asf/usergrid/blob/b5144d05/stack/tools/src/main/java/org/apache/usergrid/tools/ExportAdmins.java ---------------------------------------------------------------------- diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/ExportAdmins.java b/stack/tools/src/main/java/org/apache/usergrid/tools/ExportAdmins.java index 0bb74ab..bb6e863 100644 --- a/stack/tools/src/main/java/org/apache/usergrid/tools/ExportAdmins.java +++ b/stack/tools/src/main/java/org/apache/usergrid/tools/ExportAdmins.java @@ -23,8 +23,10 @@ import org.apache.commons.cli.Option; import org.apache.commons.cli.OptionBuilder; import org.apache.commons.cli.Options; import org.apache.usergrid.corepersistence.util.CpNamingUtils; +import org.apache.usergrid.management.UserInfo; import org.apache.usergrid.persistence.Entity; import org.apache.usergrid.persistence.EntityManager; +import org.apache.usergrid.persistence.Query; import org.apache.usergrid.persistence.Results; import org.apache.usergrid.utils.StringUtils; import org.codehaus.jackson.JsonGenerator; @@ -345,15 +347,15 @@ public class ExportAdmins extends ExportingToolBase { 0 : task.dictionariesByName.get( "credentials" )); logger.error( "{} admin user {}:{}:{} has organizations={} dictionaries={} credentials={}", - new Object[]{ - actionTaken, - task.adminUser.getProperty( "username" ), - task.adminUser.getProperty( "email" ), - task.adminUser.getUuid(), - task.orgNamesByUuid.size(), - task.dictionariesByName.size(), - creds == null ? 0 : creds.size() - } ); + new Object[]{ + actionTaken, + task.adminUser.getProperty( "username" ), + task.adminUser.getProperty( "email" ), + task.adminUser.getUuid(), + task.orgNamesByUuid.size(), + task.dictionariesByName.size(), + creds == null ? 0 : creds.size() + } ); } catch ( Exception e ) { logger.error("Error reading data for user " + uuid, e ); @@ -385,13 +387,9 @@ public class ExportAdmins extends ExportingToolBase { task.orgNamesByUuid = managementService.getOrganizationsForAdminUser( task.adminUser.getUuid() ); -<<<<<<< HEAD - List<Org> orgs = orgMap.get( task.adminUser.getProperty( "username" ).toString().toLowerCase() ); -======= List<Org> orgs = userToOrgsMap.get( task.adminUser.getProperty( "username" ).toString().toLowerCase() ); ->>>>>>> master if ( orgs != null && task.orgNamesByUuid.size() < orgs.size() ) { // list of orgs from getOrganizationsForAdminUser() is less than expected, use userToOrgsMap @@ -401,16 +399,6 @@ public class ExportAdmins extends ExportingToolBase { } task.orgNamesByUuid = bimap; } -<<<<<<< HEAD - - if ( task.orgNamesByUuid.isEmpty() ) { - logger.error("{}:{}:{} has no orgs", new Object[] { - task.adminUser.getProperty("username"), - task.adminUser.getProperty("email"), - task.adminUser.getUuid() } ); - } -======= ->>>>>>> master } } http://git-wip-us.apache.org/repos/asf/usergrid/blob/b5144d05/stack/tools/src/main/java/org/apache/usergrid/tools/ExportApp.java ---------------------------------------------------------------------- diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/ExportApp.java b/stack/tools/src/main/java/org/apache/usergrid/tools/ExportApp.java index db975e6..f5e5d34 100644 --- a/stack/tools/src/main/java/org/apache/usergrid/tools/ExportApp.java +++ b/stack/tools/src/main/java/org/apache/usergrid/tools/ExportApp.java @@ -49,13 +49,13 @@ import java.util.concurrent.atomic.AtomicInteger; /** * Export all entities and connections of a Usergrid app. - * + * * Exports data files to specified directory. - * + * * Will create as many output files as there are writeThreads (by default: 10). - * + * * Will create two types of files: *.entities for Usegrird entities and *.collections for entity to entity connections. - * + * * Every line of the data files is a complete JSON object. */ public class ExportApp extends ExportingToolBase { @@ -63,7 +63,7 @@ public class ExportApp extends ExportingToolBase { static final String APPLICATION_NAME = "application"; private static final String WRITE_THREAD_COUNT = "writeThreads"; - + String applicationName; String organizationName; @@ -76,7 +76,7 @@ public class ExportApp extends ExportingToolBase { Map<Thread, JsonGenerator> entityGeneratorsByThread = new HashMap<Thread, JsonGenerator>(); Map<Thread, JsonGenerator> connectionGeneratorsByThread = new HashMap<Thread, JsonGenerator>(); - int writeThreadCount = 10; // set via CLI option; limiting write will limit output files + int writeThreadCount = 10; // set via CLI option; limiting write will limit output files @Override @@ -96,9 +96,9 @@ public class ExportApp extends ExportingToolBase { return options; } - + /** - * Tool entry point. + * Tool entry point. */ @Override public void runTool(CommandLine line) throws Exception { @@ -123,7 +123,7 @@ public class ExportApp extends ExportingToolBase { startSpring(); - UUID applicationId = emf.lookupApplication( applicationName ); + UUID applicationId = emf.lookupApplication( applicationName ).get(); if (applicationId == null) { throw new RuntimeException( "Cannot find application " + applicationName ); } @@ -134,7 +134,7 @@ public class ExportApp extends ExportingToolBase { writeScheduler = Schedulers.from( writeThreadPoolExecutor ); Observable<String> collectionsObservable = Observable.create( new CollectionsObservable( em ) ); - + collectionsObservable.flatMap( new Func1<String, Observable<ExportEntity>>() { public Observable<ExportEntity> call(String collection) { @@ -155,24 +155,24 @@ public class ExportApp extends ExportingToolBase { .doOnCompleted( new FileWrapUpAction() ) .toBlocking().last(); } - - + + // ---------------------------------------------------------------------------------------- // reading data - + /** * Emits collection names found in application. */ class CollectionsObservable implements rx.Observable.OnSubscribe<String> { EntityManager em; - + public CollectionsObservable(EntityManager em) { this.em = em; } public void call(Subscriber<? super String> subscriber) { - + int count = 0; try { Map<String, Object> collectionMetadata = em.getApplicationCollectionMetadata(); @@ -180,17 +180,17 @@ public class ExportApp extends ExportingToolBase { subscriber.onNext( collection ); count++; } - + } catch (Exception e) { subscriber.onError( e ); } - + subscriber.onCompleted(); logger.info( "Completed. Read {} collection names", count ); } } - + /** * Emits entities of collection. */ @@ -206,9 +206,9 @@ public class ExportApp extends ExportingToolBase { public void call(Subscriber<? super ExportEntity> subscriber) { logger.info("Starting to read entities of collection {}", collection); - + subscriber.onStart(); - + try { int count = 0; @@ -229,16 +229,16 @@ public class ExportApp extends ExportingToolBase { } dictionariesByName.put( dictionary, dict ); } - - ExportEntity exportEntity = new ExportEntity( - organizationName, - applicationName, - entity, + + ExportEntity exportEntity = new ExportEntity( + organizationName, + applicationName, + entity, dictionariesByName ); - + subscriber.onNext( exportEntity ); count++; - + } catch (Exception e) { logger.error("Error reading entity " + entity.getUuid() +" from collection " + collection); } @@ -252,14 +252,14 @@ public class ExportApp extends ExportingToolBase { subscriber.onCompleted(); logger.info("Completed collection {}. Read {} entities", collection, count); - + } catch ( Exception e ) { subscriber.onError(e); } } } - + /** * Emits connections of an entity. */ @@ -276,51 +276,51 @@ public class ExportApp extends ExportingToolBase { logger.info( "Starting to read connections for entity {} type {}", exportEntity.getEntity().getName(), exportEntity.getEntity().getType() ); - + int count = 0; - + try { Set<String> connectionTypes = em.getConnectionTypes( exportEntity.getEntity() ); for (String connectionType : connectionTypes) { - Results results = em.getConnectedEntities( - exportEntity.getEntity().getUuid(), connectionType, null, Results.Level.CORE_PROPERTIES ); + Results results = em.getTargetEntities( + exportEntity.getEntity(), connectionType, null, Query.Level.CORE_PROPERTIES ); for (Entity connectedEntity : results.getEntities()) { try { - - ExportConnection connection = new ExportConnection( + + ExportConnection connection = new ExportConnection( applicationName, organizationName, - connectionType, - exportEntity.getEntity().getUuid(), + connectionType, + exportEntity.getEntity().getUuid(), connectedEntity.getUuid()); - + subscriber.onNext( connection ); count++; } catch (Exception e) { - logger.error( "Error reading connection entity " + logger.error( "Error reading connection entity " + exportEntity.getEntity().getUuid() + " -> " + connectedEntity.getType()); } } } - + } catch (Exception e) { subscriber.onError( e ); } - + subscriber.onCompleted(); logger.info("Completed entity {} type {} connections count {}", new Object[] { exportEntity.getEntity().getName(), exportEntity.getEntity().getType(), count }); } } - + // ---------------------------------------------------------------------------------------- // writing data - - + + /** * Writes entities to JSON file. */ @@ -358,7 +358,7 @@ public class ExportApp extends ExportingToolBase { } } - + /** * Writes connection to JSON file. */ @@ -396,7 +396,7 @@ public class ExportApp extends ExportingToolBase { } } - + private class FileWrapUpAction implements Action0 { @Override public void call() { http://git-wip-us.apache.org/repos/asf/usergrid/blob/b5144d05/stack/tools/src/main/java/org/apache/usergrid/tools/ExportDataCreator.java ---------------------------------------------------------------------- diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/ExportDataCreator.java b/stack/tools/src/main/java/org/apache/usergrid/tools/ExportDataCreator.java index 6fa4896..4b9e5a0 100644 --- a/stack/tools/src/main/java/org/apache/usergrid/tools/ExportDataCreator.java +++ b/stack/tools/src/main/java/org/apache/usergrid/tools/ExportDataCreator.java @@ -16,7 +16,6 @@ */ package org.apache.usergrid.tools; - import io.codearte.jfairy.Fairy; import io.codearte.jfairy.producer.company.Company; import io.codearte.jfairy.producer.person.Person; @@ -189,7 +188,7 @@ public class ExportDataCreator extends ToolBase { try { EntityRef ref = em.getAlias( "company", company.name() ); Entity companyEntity = (ref == null) ? null : em.get( ref ); - + // create company if it does not exist yet if ( companyEntity == null ) { final Map<String, Object> companyMap = new HashMap<String, Object>() {{ @@ -213,7 +212,7 @@ public class ExportDataCreator extends ToolBase { logger.error("Error creating or connecting company", e); continue; } - + try { for (int j = 0; j < 5; j++) { Activity activity = new Activity(); @@ -236,7 +235,7 @@ public class ExportDataCreator extends ToolBase { } } } - + } catch (Exception e) { logger.error("Error creating activities", e); continue; http://git-wip-us.apache.org/repos/asf/usergrid/blob/b5144d05/stack/tools/src/main/java/org/apache/usergrid/tools/ImportAdmins.java ---------------------------------------------------------------------- diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/ImportAdmins.java b/stack/tools/src/main/java/org/apache/usergrid/tools/ImportAdmins.java index f39ef9b..c8c4d99 100644 --- a/stack/tools/src/main/java/org/apache/usergrid/tools/ImportAdmins.java +++ b/stack/tools/src/main/java/org/apache/usergrid/tools/ImportAdmins.java @@ -17,17 +17,16 @@ package org.apache.usergrid.tools; -import com.sun.org.apache.bcel.internal.generic.DUP; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.Option; import org.apache.commons.cli.OptionBuilder; import org.apache.commons.cli.Options; import org.apache.commons.io.filefilter.PrefixFileFilter; +import org.apache.usergrid.corepersistence.util.CpNamingUtils; import org.apache.usergrid.management.OrganizationInfo; import org.apache.usergrid.management.UserInfo; import org.apache.usergrid.persistence.EntityManager; import org.apache.usergrid.persistence.EntityRef; -import org.apache.usergrid.persistence.Identifier; import org.apache.usergrid.persistence.SimpleEntityRef; import org.apache.usergrid.persistence.entities.User; import org.apache.usergrid.persistence.exceptions.DuplicateUniquePropertyExistsException; @@ -39,7 +38,10 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.File; -import java.util.*; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; import java.util.concurrent.BlockingQueue; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.TimeUnit; @@ -47,18 +49,18 @@ import java.util.concurrent.atomic.AtomicInteger; import static org.apache.usergrid.persistence.Schema.PROPERTY_TYPE; import static org.apache.usergrid.persistence.Schema.PROPERTY_UUID; -import static org.apache.usergrid.persistence.cassandra.CassandraService.MANAGEMENT_APPLICATION_ID; +import org.apache.usergrid.persistence.index.query.Identifier; /** * Import Admin Users and metadata including organizations and passwords. - * - * Usage Example: - * + * + * Usage Example: + * * java -Xmx8000m -Dlog4j.configuration=file:/home/me/log4j.properties -classpath . \ * -jar usergrid-tools-1.0.2.jar ImportAdmins -writeThreads 100 -auditThreads 100 \ - * -host casshost -inputDir=/home/me/import-data - * + * -host casshost -inputDir=/home/me/import-data + * * If you want to provide any property overrides, put properties file named usergrid-custom-tools.properties * in the same directory where you run the above command. For example, you might want to set the Cassandra * client threads and import to a specific set of keyspaces: @@ -97,8 +99,8 @@ public class ImportAdmins extends ToolBase { AtomicInteger writeEmptyCount = new AtomicInteger( 0 ); AtomicInteger auditEmptyCount = new AtomicInteger( 0 ); AtomicInteger metadataEmptyCount = new AtomicInteger( 0 ); - - + + static class DuplicateUser { String email; String username; @@ -110,7 +112,7 @@ public class ImportAdmins extends ToolBase { } } } - + @Override @@ -269,7 +271,7 @@ public class ImportAdmins extends ToolBase { long timeRemaining = (long) ( sizeLast / (rateAverageSum / iterations) ); logger.info("++PROGRESS ({}): sizeLast={} nowSize={} processed={} rateLast={}/s rateAvg={}/s timeRemaining={}s", - new Object[] { + new Object[] { identifier, sizeLast, sizeNow, processed, rateLast, (rateAverageSum / iterations), timeRemaining } ); } @@ -355,7 +357,7 @@ public class ImportAdmins extends ToolBase { workerThread.start(); metadataWorkerThreadMap.put(worker, workerThread); } - + logger.info( "Started {} metadata workers", writeThreadCount ); } @@ -363,7 +365,7 @@ public class ImportAdmins extends ToolBase { @SuppressWarnings("unchecked") private void importMetadata(String fileName, int writeThreads) throws Exception { - EntityManager em = emf.getEntityManager(MANAGEMENT_APPLICATION_ID); + EntityManager em = emf.getEntityManager( CpNamingUtils.MANAGEMENT_APPLICATION_ID); File metadataFile = new File(importDir, fileName); @@ -401,10 +403,10 @@ public class ImportAdmins extends ToolBase { try { EntityRef entityRef = new SimpleEntityRef( "user", UUID.fromString( entityOwnerId ) ); Map<String, Object> metadata = (Map<String, Object>) jp.readValueAs( Map.class ); - + workQueue.put( new ImportMetadataTask( entityRef, metadata ) ); logger.debug( "Put user {} in metadata queue", entityRef.getUuid() ); - + } catch ( Exception e ) { logger.debug( "Error with user {}, not putting in metadata queue", entityOwnerId ); } @@ -426,7 +428,7 @@ public class ImportAdmins extends ToolBase { EntityManager em, EntityRef entityRef, Map<String, Object> metadata) throws Exception { DuplicateUser dup = dupsByDupUuid.get( entityRef.getUuid() ); - + if ( dup == null ) { // not a duplicate User user = em.get( entityRef, User.class ); @@ -476,7 +478,7 @@ public class ImportAdmins extends ToolBase { } } } - + Map<String, Object> dictionariesMap = (Map<String, Object>) metadata.get("dictionaries"); if (dictionariesMap != null && !dictionariesMap.isEmpty()) { for (String name : dictionariesMap.keySet()) { @@ -501,17 +503,17 @@ public class ImportAdmins extends ToolBase { } else { logger.warn("User {} has no dictionaries", entityRef.getUuid() ); } - + } else { // this is a duplicate user, so merge orgs logger.info("Processing duplicate username={} email={}", dup.email, dup.username ); - - Identifier identifier = dup.email != null ? + + Identifier identifier = dup.email != null ? Identifier.fromEmail( dup.email ) : Identifier.from( dup.username ); User originalUser = em.get( em.getUserByIdentifier(identifier), User.class ); // get map of original user's orgs - + UserInfo originalUserInfo = managementService.getAdminUserByEmail( originalUser.getEmail() ); Map<String, Object> originalUserOrgData = managementService.getAdminUserOrganizationData( originalUser.getUuid() ); @@ -530,7 +532,7 @@ public class ImportAdmins extends ToolBase { if (originalUserOrgs.get( orgName ) == null) { // original user does not have this org OrganizationInfo orgInfo = managementService.getOrganizationByUuid( orgUuid ); - + if (orgInfo == null) { // org does not exist yet, create it and add original user to it try { managementService.createOrganization( orgUuid, orgName, originalUserInfo, false ); @@ -539,9 +541,9 @@ public class ImportAdmins extends ToolBase { logger.debug( "Created new org {} for user {}:{}:{} from duplicate user {}:{}", new Object[]{ orgInfo.getName(), - originalUser.getUsername(), + originalUser.getUsername(), originalUser.getEmail(), - originalUser.getUuid(), + originalUser.getUuid(), dup.username, dup.email }); @@ -554,20 +556,20 @@ public class ImportAdmins extends ToolBase { logger.debug( "Added to org user {}:{}:{} from duplicate user {}:{}", new Object[]{ orgInfo.getName(), - originalUser.getUsername(), + originalUser.getUsername(), originalUser.getEmail(), - originalUser.getUuid(), + originalUser.getUuid(), dup.username, dup.email }); } catch (Exception e) { - logger.error( "Error Adding user {} to org {}", + logger.error( "Error Adding user {} to org {}", new Object[]{originalUserInfo.getEmail(), orgName} ); } } - + } // else original user already has this org - + } } } @@ -612,7 +614,7 @@ public class ImportAdmins extends ToolBase { public void run() { int count = 0; - EntityManager em = emf.getEntityManager(MANAGEMENT_APPLICATION_ID); + EntityManager em = emf.getEntityManager(CpNamingUtils.MANAGEMENT_APPLICATION_ID); long durationSum = 0; @@ -647,7 +649,7 @@ public class ImportAdmins extends ToolBase { durationSum += duration; //logger.debug( "Audited {}th admin", userCount ); - + if ( count % 100 == 0 ) { logger.info( "Audited {}. Average Audit Rate: {}(ms)", count, durationSum / count ); } @@ -690,7 +692,7 @@ public class ImportAdmins extends ToolBase { public void run() { int count = 0; - EntityManager em = emf.getEntityManager(MANAGEMENT_APPLICATION_ID); + EntityManager em = emf.getEntityManager(CpNamingUtils.MANAGEMENT_APPLICATION_ID); long durationSum = 0; @@ -705,20 +707,20 @@ public class ImportAdmins extends ToolBase { continue; } metadataEmptyCount.set( 0 ); - + long startTime = System.currentTimeMillis(); - + importEntityMetadata( em, task.entityRef, task.metadata ); - + long stopTime = System.currentTimeMillis(); long duration = stopTime - startTime; durationSum += duration; metadataCount.getAndIncrement(); count++; - + if ( count % 30 == 0 ) { logger.info( "Imported {} metadata of total {} expected. " + - "Average metadata Imported Rate: {}(ms)", + "Average metadata Imported Rate: {}(ms)", new Object[] { metadataCount.get(), userCount.get(), durationSum / count }); } @@ -752,7 +754,7 @@ public class ImportAdmins extends ToolBase { public void run() { int count = 0; - EntityManager em = emf.getEntityManager(MANAGEMENT_APPLICATION_ID); + EntityManager em = emf.getEntityManager(CpNamingUtils.MANAGEMENT_APPLICATION_ID); long durationSum = 0; @@ -776,7 +778,7 @@ public class ImportAdmins extends ToolBase { try { long startTime = System.currentTimeMillis(); - + em.create(uuid, type, entityProps); logger.debug( "Imported admin user {}:{}:{}", @@ -787,20 +789,20 @@ public class ImportAdmins extends ToolBase { long stopTime = System.currentTimeMillis(); long duration = stopTime - startTime; durationSum += duration; - + count++; if (count % 30 == 0) { logger.info( "This worked has imported {} users of total {} imported so far. " + - "Average Creation Rate: {}ms", + "Average Creation Rate: {}ms", new Object[] { count, userCount.get(), durationSum / count }); } - + } catch (DuplicateUniquePropertyExistsException de) { String dupProperty = de.getPropertyName(); handleDuplicateAccount( em, dupProperty, entityProps ); continue; - + } catch (Exception e) { logger.error("Error", e); } @@ -811,19 +813,19 @@ public class ImportAdmins extends ToolBase { } } - + private void handleDuplicateAccount(EntityManager em, String dupProperty, Map<String, Object> entityProps ) { logger.info( "Processing duplicate user {}:{}:{} with duplicate {}", new Object[]{ - entityProps.get( "username" ), - entityProps.get( "email" ), - entityProps.get( "uuid" ), + entityProps.get( "username" ), + entityProps.get( "email" ), + entityProps.get( "uuid" ), dupProperty} ); - + UUID dupUuid = UUID.fromString( entityProps.get("uuid").toString() ); try { dupsByDupUuid.put( dupUuid, new DuplicateUser( dupProperty, entityProps ) ); - + } catch (Exception e) { logger.error("Error processing dup user {}:{}:{}", new Object[] {entityProps.get( "username" ), entityProps.get("email"), dupUuid}); http://git-wip-us.apache.org/repos/asf/usergrid/blob/b5144d05/stack/tools/src/main/java/org/apache/usergrid/tools/Metrics.java ---------------------------------------------------------------------- diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/Metrics.java b/stack/tools/src/main/java/org/apache/usergrid/tools/Metrics.java index b308290..a056f77 100644 --- a/stack/tools/src/main/java/org/apache/usergrid/tools/Metrics.java +++ b/stack/tools/src/main/java/org/apache/usergrid/tools/Metrics.java @@ -26,6 +26,7 @@ import java.util.Map; import java.util.Set; import java.util.UUID; +import org.apache.usergrid.persistence.index.query.CounterResolution; import org.codehaus.jackson.JsonGenerator; import org.apache.usergrid.management.ApplicationInfo; import org.apache.usergrid.management.OrganizationInfo; @@ -46,7 +47,6 @@ import com.google.common.collect.ArrayListMultimap; import com.google.common.collect.BiMap; import com.google.common.collect.ListMultimap; import com.google.common.collect.Ordering; -import org.apache.usergrid.persistence.index.query.CounterResolution; /** http://git-wip-us.apache.org/repos/asf/usergrid/blob/b5144d05/stack/tools/src/main/java/org/apache/usergrid/tools/OrganizationExport.java ---------------------------------------------------------------------- diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/OrganizationExport.java b/stack/tools/src/main/java/org/apache/usergrid/tools/OrganizationExport.java index 4fbccb2..4493b67 100644 --- a/stack/tools/src/main/java/org/apache/usergrid/tools/OrganizationExport.java +++ b/stack/tools/src/main/java/org/apache/usergrid/tools/OrganizationExport.java @@ -25,7 +25,7 @@ import java.util.UUID; import org.apache.usergrid.management.UserInfo; import org.apache.usergrid.persistence.Entity; import org.apache.usergrid.persistence.EntityManager; -import org.apache.usergrid.persistence.index.query.Query; +import org.apache.usergrid.persistence.Query; import org.apache.usergrid.persistence.Results; import org.apache.commons.cli.CommandLine; http://git-wip-us.apache.org/repos/asf/usergrid/blob/b5144d05/stack/tools/src/main/java/org/apache/usergrid/tools/PopulateSample.java ---------------------------------------------------------------------- diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/PopulateSample.java b/stack/tools/src/main/java/org/apache/usergrid/tools/PopulateSample.java index b67e9eb..48e2af6 100644 --- a/stack/tools/src/main/java/org/apache/usergrid/tools/PopulateSample.java +++ b/stack/tools/src/main/java/org/apache/usergrid/tools/PopulateSample.java @@ -46,7 +46,7 @@ import static org.apache.usergrid.services.ServicePayload.payload; public class PopulateSample extends ToolBase { - private static final Logger logger = LoggerFactory.getLogger( Export.class ); + private static final Logger logger = LoggerFactory.getLogger( PopulateSample.class ); @Override http://git-wip-us.apache.org/repos/asf/usergrid/blob/b5144d05/stack/tools/src/main/java/org/apache/usergrid/tools/UserManager.java ---------------------------------------------------------------------- diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/UserManager.java b/stack/tools/src/main/java/org/apache/usergrid/tools/UserManager.java index 3b5383d..e4195a4 100644 --- a/stack/tools/src/main/java/org/apache/usergrid/tools/UserManager.java +++ b/stack/tools/src/main/java/org/apache/usergrid/tools/UserManager.java @@ -18,6 +18,7 @@ package org.apache.usergrid.tools; import com.google.common.collect.BiMap; +import org.apache.usergrid.corepersistence.util.CpNamingUtils; import org.apache.usergrid.management.UserInfo; import org.apache.commons.cli.CommandLine; @@ -65,7 +66,7 @@ public class UserManager extends ToolBase { logger.info( mapToFormattedJsonString( orgs ) ); logger.info("--- User dictionaries:"); - EntityManager em = emf.getEntityManager( CassandraService.MANAGEMENT_APPLICATION_ID ); + EntityManager em = emf.getEntityManager( CpNamingUtils.MANAGEMENT_APPLICATION_ID ); User user = em.get( userInfo.getUuid(), User.class ); Set<String> dictionaries = em.getDictionaries( user ); for (String dictionary : dictionaries) { http://git-wip-us.apache.org/repos/asf/usergrid/blob/b5144d05/stack/tools/src/main/java/org/apache/usergrid/tools/WarehouseExport.java ---------------------------------------------------------------------- diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/WarehouseExport.java b/stack/tools/src/main/java/org/apache/usergrid/tools/WarehouseExport.java index 7b60257..564bd7e 100644 --- a/stack/tools/src/main/java/org/apache/usergrid/tools/WarehouseExport.java +++ b/stack/tools/src/main/java/org/apache/usergrid/tools/WarehouseExport.java @@ -17,63 +17,42 @@ package org.apache.usergrid.tools; +import au.com.bytecode.opencsv.CSVReader; +import au.com.bytecode.opencsv.CSVWriter; +import com.amazonaws.ClientConfiguration; +import com.amazonaws.Protocol; +import com.amazonaws.auth.AWSCredentials; +import com.amazonaws.auth.BasicAWSCredentials; +import com.amazonaws.services.s3.AmazonS3Client; +import com.amazonaws.services.s3.model.PutObjectResult; +import com.google.common.collect.BiMap; +import com.google.common.collect.ImmutableSet; +import com.google.inject.Module; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.OptionBuilder; +import org.apache.commons.cli.Options; +import org.apache.usergrid.management.OrganizationInfo; +import org.apache.usergrid.persistence.*; +import org.apache.usergrid.persistence.entities.Application; +import org.apache.usergrid.persistence.schema.CollectionInfo; +import org.apache.usergrid.utils.InflectionUtils; +import org.jclouds.http.config.JavaUrlHttpCommandExecutorServiceModule; +import org.jclouds.logging.log4j.config.Log4JLoggingModule; +import org.jclouds.netty.config.NettyPayloadModule; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import java.io.File; import java.io.FileReader; import java.io.FileWriter; import java.io.FilenameFilter; import java.text.SimpleDateFormat; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.Date; -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Map; +import java.util.*; import java.util.Map.Entry; -import java.util.Properties; -import java.util.Set; -import java.util.TreeMap; -import java.util.UUID; import java.util.concurrent.TimeUnit; -import org.jclouds.ContextBuilder; -import org.jclouds.blobstore.AsyncBlobStore; -import org.jclouds.blobstore.BlobStoreContext; -import org.jclouds.blobstore.domain.Blob; -import org.jclouds.blobstore.domain.BlobBuilder; -import org.jclouds.blobstore.options.PutOptions; -import org.jclouds.http.config.JavaUrlHttpCommandExecutorServiceModule; -import org.jclouds.logging.log4j.config.Log4JLoggingModule; -import org.jclouds.netty.config.NettyPayloadModule; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.apache.usergrid.management.OrganizationInfo; -import org.apache.usergrid.persistence.Entity; -import org.apache.usergrid.persistence.EntityManager; -import org.apache.usergrid.persistence.index.query.Query; -import org.apache.usergrid.persistence.Results; -import org.apache.usergrid.persistence.Schema; -import org.apache.usergrid.persistence.entities.Application; -import org.apache.usergrid.persistence.schema.CollectionInfo; -import org.apache.usergrid.utils.InflectionUtils; - -import org.apache.commons.cli.CommandLine; -import org.apache.commons.cli.Option; -import org.apache.commons.cli.OptionBuilder; -import org.apache.commons.cli.Options; - -import com.google.common.collect.BiMap; -import com.google.common.collect.ImmutableSet; -import com.google.common.util.concurrent.ListenableFuture; -import com.google.inject.Module; - -import au.com.bytecode.opencsv.CSVReader; -import au.com.bytecode.opencsv.CSVWriter; - import static org.apache.usergrid.persistence.Schema.getDefaultSchema; -import org.apache.usergrid.persistence.index.query.Query.Level; /** @@ -195,40 +174,16 @@ public class WarehouseExport extends ExportingToolBase { .of( new JavaUrlHttpCommandExecutorServiceModule(), new Log4JLoggingModule(), new NettyPayloadModule() ); - BlobStoreContext context = - ContextBuilder.newBuilder( "s3" ).credentials( accessId, secretKey ).modules( MODULES ) - .overrides( overrides ).buildView( BlobStoreContext.class ); - - // Create Container (the bucket in s3) - try { - AsyncBlobStore blobStore = context.getAsyncBlobStore(); // it can be changed to sync - // BlobStore (returns false if it already exists) - ListenableFuture<Boolean> container = blobStore.createContainerInLocation( null, bucketName ); - if ( container.get() ) { - LOG.info( "Created bucket " + bucketName ); - } - } - catch ( Exception ex ) { - logger.error( "Could not start binary service: {}", ex.getMessage() ); - throw new RuntimeException( ex ); - } - - try { - File file = new File( fileName ); - AsyncBlobStore blobStore = context.getAsyncBlobStore(); - BlobBuilder blobBuilder = - blobStore.blobBuilder( file.getName() ).payload( file ).calculateMD5().contentType( "text/plain" ) - .contentLength( file.length() ); - - Blob blob = blobBuilder.build(); + AWSCredentials credentials = new BasicAWSCredentials(accessId, secretKey); + ClientConfiguration clientConfig = new ClientConfiguration(); + clientConfig.setProtocol( Protocol.HTTP); - ListenableFuture<String> futureETag = blobStore.putBlob( bucketName, blob, PutOptions.Builder.multipart() ); + AmazonS3Client s3Client = new AmazonS3Client(credentials, clientConfig); - LOG.info( "Uploaded file etag=" + futureETag.get() ); - } - catch ( Exception e ) { - LOG.error( "Error uploading to blob store", e ); - } + s3Client.createBucket( bucketName ); + File uploadFile = new File( fileName ); + PutObjectResult putObjectResult = s3Client.putObject( bucketName, uploadFile.getName(), uploadFile ); + LOG.info("Uploaded file etag={}", putObjectResult.getETag()); } @@ -453,7 +408,7 @@ public class WarehouseExport extends ExportingToolBase { Query query = Query.fromQL( queryString ); query.setLimit( MAX_ENTITY_FETCH ); - query.setResultsLevel( Level.REFS ); + query.setResultsLevel( Query.Level.REFS ); Results results = em.searchCollection( em.getApplicationRef(), collectionName, query ); while ( results.size() > 0 ) { http://git-wip-us.apache.org/repos/asf/usergrid/blob/b5144d05/stack/tools/src/main/java/org/apache/usergrid/tools/bean/MetricQuery.java ---------------------------------------------------------------------- diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/bean/MetricQuery.java b/stack/tools/src/main/java/org/apache/usergrid/tools/bean/MetricQuery.java index b0f4f6e..8402dd2 100644 --- a/stack/tools/src/main/java/org/apache/usergrid/tools/bean/MetricQuery.java +++ b/stack/tools/src/main/java/org/apache/usergrid/tools/bean/MetricQuery.java @@ -21,11 +21,7 @@ import java.util.ArrayList; import java.util.List; import java.util.UUID; -import org.apache.usergrid.persistence.AggregateCounter; -import org.apache.usergrid.persistence.AggregateCounterSet; -import org.apache.usergrid.persistence.EntityManager; -import org.apache.usergrid.persistence.index.query.Query; -import org.apache.usergrid.persistence.Results; +import org.apache.usergrid.persistence.*; import com.google.common.base.Preconditions; import org.apache.usergrid.persistence.index.query.CounterResolution; http://git-wip-us.apache.org/repos/asf/usergrid/blob/b5144d05/stack/tools/src/test/java/org/apache/usergrid/tools/ExportAppTest.java ---------------------------------------------------------------------- diff --git a/stack/tools/src/test/java/org/apache/usergrid/tools/ExportAppTest.java b/stack/tools/src/test/java/org/apache/usergrid/tools/ExportAppTest.java index 446aa91..890c972 100644 --- a/stack/tools/src/test/java/org/apache/usergrid/tools/ExportAppTest.java +++ b/stack/tools/src/test/java/org/apache/usergrid/tools/ExportAppTest.java @@ -19,25 +19,12 @@ package org.apache.usergrid.tools; import org.apache.commons.lang.RandomStringUtils; import org.apache.usergrid.ServiceITSetup; import org.apache.usergrid.ServiceITSetupImpl; -import org.apache.usergrid.ServiceITSuite; -import org.apache.usergrid.management.ApplicationInfo; -import org.apache.usergrid.management.OrganizationOwnerInfo; -import org.apache.usergrid.persistence.Entity; -import org.apache.usergrid.persistence.EntityManager; import org.junit.ClassRule; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import rx.Scheduler; -import rx.schedulers.Schedulers; import java.io.File; import java.io.FileFilter; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.atomic.AtomicInteger; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -48,45 +35,45 @@ import static org.junit.Assert.assertTrue; */ public class ExportAppTest { static final Logger logger = LoggerFactory.getLogger( ExportAppTest.class ); - + int NUM_COLLECTIONS = 10; - int NUM_ENTITIES = 50; + int NUM_ENTITIES = 50; int NUM_CONNECTIONS = 3; @ClassRule - public static ServiceITSetup setup = new ServiceITSetupImpl( ServiceITSuite.cassandraResource ); + public static ServiceITSetup setup = new ServiceITSetupImpl(); @org.junit.Test public void testBasicOperation() throws Exception { - + String rand = RandomStringUtils.randomAlphanumeric( 10 ); - + // create app with some data String orgName = "org_" + rand; String appName = "app_" + rand; - + ExportDataCreator creator = new ExportDataCreator(); creator.startTool( new String[] { "-organization", orgName, "-application", appName, - "-host", "localhost:" + ServiceITSuite.cassandraResource.getRpcPort() + "-host", "localhost:9120" }, false); - + long start = System.currentTimeMillis(); - + String directoryName = "target/export" + rand; ExportApp exportApp = new ExportApp(); exportApp.startTool( new String[]{ "-application", orgName + "/" + appName, "-writeThreads", "100", - "-host", "localhost:" + ServiceITSuite.cassandraResource.getRpcPort(), + "-host", "localhost:9120", "-outputDir", directoryName }, false ); logger.info( "100 read and 100 write threads = " + (System.currentTimeMillis() - start) / 1000 + "s" ); - + File exportDir = new File(directoryName); assertTrue( getFileCount( exportDir, "entities" ) > 0 ); assertTrue( getFileCount( exportDir, "connections" ) > 0 ); @@ -97,7 +84,7 @@ public class ExportAppTest { exportApp.startTool( new String[]{ "-application", orgName + "/" + appName, "-writeThreads", "1", - "-host", "localhost:" + ServiceITSuite.cassandraResource.getRpcPort(), + "-host", "localhost:9120", "-outputDir", directoryName + "1" }, false ); @@ -115,4 +102,4 @@ public class ExportAppTest { } } ).length; } -} \ No newline at end of file +} http://git-wip-us.apache.org/repos/asf/usergrid/blob/b5144d05/stack/tools/src/test/java/org/apache/usergrid/tools/ExportImportAdminsTest.java ---------------------------------------------------------------------- diff --git a/stack/tools/src/test/java/org/apache/usergrid/tools/ExportImportAdminsTest.java b/stack/tools/src/test/java/org/apache/usergrid/tools/ExportImportAdminsTest.java index 9cce040..23ff096 100644 --- a/stack/tools/src/test/java/org/apache/usergrid/tools/ExportImportAdminsTest.java +++ b/stack/tools/src/test/java/org/apache/usergrid/tools/ExportImportAdminsTest.java @@ -22,7 +22,7 @@ import org.apache.commons.io.IOUtils; import org.apache.commons.lang.RandomStringUtils; import org.apache.usergrid.ServiceITSetup; import org.apache.usergrid.ServiceITSetupImpl; -import org.apache.usergrid.ServiceITSuite; +import org.apache.usergrid.corepersistence.util.CpNamingUtils; import org.apache.usergrid.management.OrganizationInfo; import org.apache.usergrid.management.OrganizationOwnerInfo; import org.apache.usergrid.management.UserInfo; @@ -39,18 +39,20 @@ import java.io.File; import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.FilenameFilter; -import java.util.*; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.UUID; import static junit.framework.TestCase.assertNotNull; -import static org.apache.usergrid.persistence.cassandra.CassandraService.MANAGEMENT_APPLICATION_ID; import static org.junit.Assert.*; public class ExportImportAdminsTest { static final Logger logger = LoggerFactory.getLogger( ExportImportAdminsTest.class ); - + @ClassRule - public static ServiceITSetup setup = new ServiceITSetupImpl( ServiceITSuite.cassandraResource ); + public static ServiceITSetup setup = new ServiceITSetupImpl(); @org.junit.Test public void testExportUserAndOrg() throws Exception { @@ -81,7 +83,7 @@ public class ExportImportAdminsTest { ExportAdmins exportAdmins = new ExportAdmins(); exportAdmins.startTool( new String[] { - "-host", "localhost:" + ServiceITSuite.cassandraResource.getRpcPort(), + "-host", "localhost:9120", "-outputDir", directoryName }, false ); @@ -148,7 +150,7 @@ public class ExportImportAdminsTest { public void testImportAdminUsersAndOrgs() throws Exception { // first: generate the data file with unique user and org IDs and names - + // data contains three users each with a unique org, one user has a duplicate email String rand1 = RandomStringUtils.randomAlphanumeric( 10 ); @@ -214,7 +216,7 @@ public class ExportImportAdminsTest { ImportAdmins importAdmins = new ImportAdmins(); importAdmins.startTool( new String[]{ - "-host", "localhost:" + ServiceITSuite.cassandraResource.getRpcPort(), + "-host", "localhost:9120", "-inputDir", tempDir.getAbsolutePath() }, false ); @@ -229,7 +231,7 @@ public class ExportImportAdminsTest { assertNotNull( "org 2 exists", orgInfo2 ); List<UserInfo> org2_users = setup.getMgmtSvc().getAdminUsersForOrganization( org_uuid_2 ); assertEquals( "org2 has two users", 2, org2_users.size() ); - + OrganizationInfo orgInfo3 = setup.getMgmtSvc().getOrganizationByUuid( org_uuid_3 ); assertNotNull( "org 3 exists", orgInfo3 ); List<UserInfo> org3_users = setup.getMgmtSvc().getAdminUsersForOrganization( org_uuid_3 ); @@ -237,7 +239,7 @@ public class ExportImportAdminsTest { BiMap<UUID, String> user1_orgs = setup.getMgmtSvc().getOrganizationsForAdminUser( user_uuid_1 ); assertEquals( "user 1 has 2 orgs", 2, user1_orgs.size() ); - + BiMap<UUID, String> user2_orgs = setup.getMgmtSvc().getOrganizationsForAdminUser( user_uuid_2 ); assertEquals( "user 2 has two orgs gained one from duplicate", 2, user2_orgs.size() ); @@ -248,10 +250,10 @@ public class ExportImportAdminsTest { logger.info("EXCEPTION EXPECTED"); } - EntityManager em = setup.getEmf().getEntityManager( MANAGEMENT_APPLICATION_ID ); + EntityManager em = setup.getEmf().getEntityManager( CpNamingUtils.MANAGEMENT_APPLICATION_ID ); Entity user3 = em.get( user_uuid_3 ); assertNull( "duplicate user does not exist", user3 ); } -} \ No newline at end of file +}
