Merge branch 'release'
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/c4813b5f Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/c4813b5f Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/c4813b5f Branch: refs/heads/master Commit: c4813b5f5ec0b9b56f53a582ddc03a19801ddb3c Parents: d796b68 62ad5a8 Author: Michael Russo <[email protected]> Authored: Thu Jan 21 16:13:03 2016 -0800 Committer: Michael Russo <[email protected]> Committed: Thu Jan 21 16:24:41 2016 -0800 ---------------------------------------------------------------------- .../main/resources/usergrid-default.properties | 10 +- .../corepersistence/CpEntityManagerFactory.java | 18 +- .../corepersistence/CpRelationManager.java | 54 ++-- .../usergrid/corepersistence/CpSetup.java | 14 +- .../read/traverse/AbstractReadGraphFilter.java | 37 ++- .../persistence/EntityManagerFactory.java | 8 +- .../usergrid/persistence/EntityManagerIT.java | 33 +++ .../graph/impl/stage/EdgeDeleteRepairImpl.java | 8 +- .../model/entity/MapToEntityConverter.java | 18 +- .../persistence/model/field/FieldTypeName.java | 3 +- .../persistence/model/field/NullField.java | 44 +++ .../model/entity/MapToEntityConverterTest.java | 28 ++ .../persistence/model/field/EntityTest.java | 9 +- .../persistence/index/impl/EntityField.java | 11 + .../index/impl/EntityMappingParser.java | 16 +- .../index/impl/EntityToMapConverter.java | 3 +- .../persistence/index/impl/FieldParser.java | 5 +- .../persistence/index/impl/IndexingUtils.java | 1 + .../persistence/index/impl/EntityIndexTest.java | 69 ++++- .../index/impl/EntityToMapConverterTest.java | 149 +++++++--- .../index/impl/IndexLoadTestsIT.java | 1 + .../management/AppInfoMigrationPlugin.java | 3 +- .../usergrid/management/ManagementService.java | 4 +- .../cassandra/ManagementServiceImpl.java | 25 +- .../migration/AppInfoMigrationPluginTest.java | 2 +- .../apache/usergrid/tools/DupAdminRepair.java | 297 ------------------- .../org/apache/usergrid/tools/DupOrgRepair.java | 263 ---------------- .../org/apache/usergrid/tools/ExportApp.java | 16 +- .../org/apache/usergrid/tools/ImportAdmins.java | 81 +---- .../apache/usergrid/tools/PopulateSample.java | 2 +- .../org/apache/usergrid/tools/ToolBase.java | 6 +- .../apache/usergrid/tools/bean/MetricQuery.java | 7 +- .../apache/usergrid/tools/ExportAppTest.java | 6 +- .../usergrid/tools/ExportImportAdminsTest.java | 3 +- 34 files changed, 485 insertions(+), 769 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/usergrid/blob/c4813b5f/stack/config/src/main/resources/usergrid-default.properties ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/usergrid/blob/c4813b5f/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java ---------------------------------------------------------------------- diff --cc stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java index 0e8a6df,8b6feed..e3a0933 --- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java +++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java @@@ -225,12 -225,10 +224,12 @@@ public class CpEntityManagerFactory imp applicationId = applicationId==null ? UUIDGenerator.newTimeUUID() : applicationId; - logger.debug( "New application orgName {} orgAppName {} id {} ", - new Object[] { orgName, name, applicationId.toString() } ); + if (logger.isDebugEnabled()) { + logger.debug("New application orgName {} orgAppName {} id {} ", + new Object[]{orgName, name, applicationId.toString()}); + } - return initializeApplicationV2( orgName, applicationId, appName, properties ); + return initializeApplicationV2( orgName, applicationId, appName, properties, forMigration); } http://git-wip-us.apache.org/repos/asf/usergrid/blob/c4813b5f/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java ---------------------------------------------------------------------- diff --cc stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java index 68f22d8,415fb5e..fa15f8f --- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java +++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java @@@ -698,32 -698,10 +700,11 @@@ public class CpRelationManager implemen indexService.queueNewEdge( applicationScope, targetEntity, edge ); - //now read all older versions of an edge, and remove them. Finally calling delete - final SearchByEdge searchByEdge = - new SimpleSearchByEdge( edge.getSourceNode(), edge.getType(), edge.getTargetNode(), Long.MAX_VALUE, - SearchByEdgeType.Order.DESCENDING, Optional.absent() ); - - - //load our versions, only retain the most recent one - gm.loadEdgeVersions(searchByEdge).skip(1).flatMap(edgeToDelete -> { - - if (logger.isDebugEnabled()) { - logger.debug("Marking edge {} for deletion", edgeToDelete); - } + // remove any duplicate edges (keeps the duplicate edge with same timestamp) + removeDuplicateEdgesAsync(gm, edge); - return gm.markEdge(edgeToDelete ); - }).lastOrDefault(null).doOnNext(lastEdge -> { - //no op if we hit our default - if (lastEdge == null) { - return; - } - - //don't queue delete b/c that de-indexes, we need to delete the edges only since we have a version still existing to index. - - gm.deleteEdge(lastEdge).toBlocking().lastOrDefault(null); // this should throw an exception - }).toBlocking().lastOrDefault(null);//this should throw an exception + return connection; } http://git-wip-us.apache.org/repos/asf/usergrid/blob/c4813b5f/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/AbstractReadGraphFilter.java ---------------------------------------------------------------------- diff --cc stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/AbstractReadGraphFilter.java index f62fee1,dd59b77..d7d6294 --- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/AbstractReadGraphFilter.java +++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/AbstractReadGraphFilter.java @@@ -175,12 -175,11 +175,13 @@@ public abstract class AbstractReadGraph return !isDeleted && !isSourceNodeDeleted && !isTargetNodeDelete; - }) + }) // any non-deleted edges should be de-duped here so the results are unique + .distinct( new EdgeDistinctKey() ) //set the edge state for cursors .doOnNext( edge -> { - logger.trace( "Seeking over edge {}", edge ); + if (logger.isTraceEnabled()) { + logger.trace("Seeking over edge {}", edge); + } edgeCursorState.update( edge ); } ) http://git-wip-us.apache.org/repos/asf/usergrid/blob/c4813b5f/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManagerFactory.java ---------------------------------------------------------------------- diff --cc stack/core/src/main/java/org/apache/usergrid/persistence/EntityManagerFactory.java index cb44d4d,2bd1ccb..751adc8 --- a/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManagerFactory.java +++ b/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManagerFactory.java @@@ -23,10 -22,8 +22,8 @@@ import java.util.UUID import com.google.common.base.Optional; - import org.apache.usergrid.corepersistence.index.ReIndexService; import org.apache.usergrid.persistence.core.util.Health; -import org.apache.usergrid.persistence.index.IndexRefreshCommand; +import org.apache.usergrid.persistence.index.EntityIndex; - import rx.Observable; /** http://git-wip-us.apache.org/repos/asf/usergrid/blob/c4813b5f/stack/core/src/test/java/org/apache/usergrid/persistence/EntityManagerIT.java ---------------------------------------------------------------------- diff --cc stack/core/src/test/java/org/apache/usergrid/persistence/EntityManagerIT.java index 0f257fb,bc693ce..cb3a728 --- a/stack/core/src/test/java/org/apache/usergrid/persistence/EntityManagerIT.java +++ b/stack/core/src/test/java/org/apache/usergrid/persistence/EntityManagerIT.java @@@ -584,4 -583,37 +584,37 @@@ public class EntityManagerIT extends Ab assertNotNull( em.get( user.getUuid() ) ); } + + + @Test + public void testFilteringOfDuplicateEdges() throws Exception { - LOG.info( "EntityManagerIT.testFilteringOfDuplicateEdges" ); ++ logger.info( "EntityManagerIT.testFilteringOfDuplicateEdges" ); + + EntityManager em = app.getEntityManager(); + + Map<String, Object> properties = new LinkedHashMap<String, Object>(); + properties.put( "name", "fluffy1" ); + + Entity entity = em.create( "fluffy", properties ); + + + EntityRef appRef = new SimpleEntityRef("application", app.getId()); + EntityRef entityRef = new SimpleEntityRef(entity.getType(), entity.getUuid()); + + assertNotNull( entity ); + + + // create duplicate edges + em.addToCollection(appRef, "fluffies", entityRef); + em.addToCollection(appRef, "fluffies", entityRef); + + //app.refreshIndex(); + + Results results = em.getCollection(appRef, + "fluffies", null, 10, Level.ALL_PROPERTIES, true); - ++ + // we should be filtering duplicate edges so only assert 1 result back and not the # of edges + assertEquals(1, results.getEntities().size()); + } + } http://git-wip-us.apache.org/repos/asf/usergrid/blob/c4813b5f/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/field/FieldTypeName.java ---------------------------------------------------------------------- diff --cc stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/field/FieldTypeName.java index 8b0cdc3,975c6bb..73f872d --- a/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/field/FieldTypeName.java +++ b/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/field/FieldTypeName.java @@@ -40,5 -40,5 +40,6 @@@ public enum FieldTypeName SET, STRING, UUID, - DISTANCE ++ DISTANCE, + NULL } http://git-wip-us.apache.org/repos/asf/usergrid/blob/c4813b5f/stack/corepersistence/model/src/test/java/org/apache/usergrid/persistence/model/entity/MapToEntityConverterTest.java ---------------------------------------------------------------------- diff --cc stack/corepersistence/model/src/test/java/org/apache/usergrid/persistence/model/entity/MapToEntityConverterTest.java index a5aa66b,03dfa11..23e6696 --- a/stack/corepersistence/model/src/test/java/org/apache/usergrid/persistence/model/entity/MapToEntityConverterTest.java +++ b/stack/corepersistence/model/src/test/java/org/apache/usergrid/persistence/model/entity/MapToEntityConverterTest.java @@@ -70,7 -70,8 +70,8 @@@ public class MapToEntityConverterTest data.put("floatField", floatField); data.put("location", coordinates); data.put("objectField", objectField); - data.put("nullField", null) -; ++ data.put("nullField", null); + // convert the map to an entity MapToEntityConverter converter = new MapToEntityConverter(); @@@ -112,4 -114,29 +114,30 @@@ assertTrue(entity.getField("parentArray").getTypeName() == FieldTypeName.LIST); } ++ + @Test + public void testNullWithinArrays() { + + // build top-level map + final Map<String,Object> data = new HashMap<>(1); + + final List<Object> arrayNullValues = new ArrayList<>(1); + arrayNullValues.add(null); + arrayNullValues.add(null); + + // add the nested list to the map + data.put("arrayNullValues", arrayNullValues); + + // convert the map to an entity + MapToEntityConverter converter = new MapToEntityConverter(); + Entity entity = converter.fromMap(data, true); + + // make sure the nested array got converted into a ListField + assertTrue(entity.getField("arrayNullValues").getTypeName() == FieldTypeName.LIST); + + List arrayReturned = (List) entity.getField("arrayNullValues").getValue(); + + assertTrue( arrayReturned.get(0) == null); + } + } http://git-wip-us.apache.org/repos/asf/usergrid/blob/c4813b5f/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/usergrid/blob/c4813b5f/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/usergrid/blob/c4813b5f/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/IndexLoadTestsIT.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/usergrid/blob/c4813b5f/stack/services/src/main/java/org/apache/usergrid/management/AppInfoMigrationPlugin.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/usergrid/blob/c4813b5f/stack/services/src/main/java/org/apache/usergrid/management/ManagementService.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/usergrid/blob/c4813b5f/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/usergrid/blob/c4813b5f/stack/services/src/test/java/org/apache/usergrid/corepersistence/migration/AppInfoMigrationPluginTest.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/usergrid/blob/c4813b5f/stack/tools/src/main/java/org/apache/usergrid/tools/ExportApp.java ---------------------------------------------------------------------- diff --cc stack/tools/src/main/java/org/apache/usergrid/tools/ExportApp.java index 9e522d2,a90a83b..9a9c522 --- a/stack/tools/src/main/java/org/apache/usergrid/tools/ExportApp.java +++ b/stack/tools/src/main/java/org/apache/usergrid/tools/ExportApp.java @@@ -142,7 -144,7 +144,8 @@@ public class ExportApp extends Exportin return Observable.create( new EntityObservable( em, collection ) ) .doOnNext( new EntityWriteAction() ).subscribeOn( writeScheduler ); - }, writeThreadCount ).flatMap( exportEntity -> { ++ + } ).flatMap( exportEntity -> { return Observable.create( new ConnectionsObservable( em, exportEntity ) ) .doOnNext( new ConnectionWriteAction() ).subscribeOn( writeScheduler ); http://git-wip-us.apache.org/repos/asf/usergrid/blob/c4813b5f/stack/tools/src/main/java/org/apache/usergrid/tools/ImportAdmins.java ---------------------------------------------------------------------- diff --cc stack/tools/src/main/java/org/apache/usergrid/tools/ImportAdmins.java index c8c4d99,c1a60b8..0c6337f --- a/stack/tools/src/main/java/org/apache/usergrid/tools/ImportAdmins.java +++ b/stack/tools/src/main/java/org/apache/usergrid/tools/ImportAdmins.java @@@ -50,8 -50,6 +50,9 @@@ import java.util.concurrent.atomic.Atom import static org.apache.usergrid.persistence.Schema.PROPERTY_TYPE; import static org.apache.usergrid.persistence.Schema.PROPERTY_UUID; +import org.apache.usergrid.persistence.index.query.Identifier; + ++ /** * Import Admin Users and metadata including organizations and passwords. * http://git-wip-us.apache.org/repos/asf/usergrid/blob/c4813b5f/stack/tools/src/main/java/org/apache/usergrid/tools/PopulateSample.java ---------------------------------------------------------------------- diff --cc stack/tools/src/main/java/org/apache/usergrid/tools/PopulateSample.java index 48e2af6,48e2af6..a9077be --- a/stack/tools/src/main/java/org/apache/usergrid/tools/PopulateSample.java +++ b/stack/tools/src/main/java/org/apache/usergrid/tools/PopulateSample.java @@@ -66,7 -66,7 +66,7 @@@ public class PopulateSample extends Too logger.info( "Starting test..." ); startSpring(); -- UserInfo user = managementService.createAdminUser( "admin", "admin", "[email protected]", "none", false, false ); ++ UserInfo user = managementService.createAdminUser( null, "admin", "admin", "[email protected]", "none", false, false ); logger.info( "Creating organization: sample-organization" ); // management http://git-wip-us.apache.org/repos/asf/usergrid/blob/c4813b5f/stack/tools/src/main/java/org/apache/usergrid/tools/bean/MetricQuery.java ---------------------------------------------------------------------- diff --cc stack/tools/src/main/java/org/apache/usergrid/tools/bean/MetricQuery.java index 8402dd2,49b110b..85582a3 --- a/stack/tools/src/main/java/org/apache/usergrid/tools/bean/MetricQuery.java +++ b/stack/tools/src/main/java/org/apache/usergrid/tools/bean/MetricQuery.java @@@ -21,12 -25,7 +25,9 @@@ import java.util.ArrayList import java.util.List; import java.util.UUID; - import org.apache.usergrid.persistence.*; - import com.google.common.base.Preconditions; - import org.apache.usergrid.persistence.index.query.CounterResolution; + + /** @author zznate */ public class MetricQuery { http://git-wip-us.apache.org/repos/asf/usergrid/blob/c4813b5f/stack/tools/src/test/java/org/apache/usergrid/tools/ExportAppTest.java ---------------------------------------------------------------------- diff --cc stack/tools/src/test/java/org/apache/usergrid/tools/ExportAppTest.java index 429e700,2946ddb..cbc06c2 --- a/stack/tools/src/test/java/org/apache/usergrid/tools/ExportAppTest.java +++ b/stack/tools/src/test/java/org/apache/usergrid/tools/ExportAppTest.java @@@ -17,9 -17,7 +17,10 @@@ package org.apache.usergrid.tools; import org.apache.commons.lang.RandomStringUtils; +import org.apache.usergrid.ServiceITSetup; +import org.apache.usergrid.ServiceITSetupImpl; +import org.junit.ClassRule; + import org.apache.usergrid.services.AbstractServiceIT; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@@ -41,9 -38,7 +42,10 @@@ public class ExportAppTest extends Abst int NUM_ENTITIES = 50; int NUM_CONNECTIONS = 3; + @ClassRule + public static ServiceITSetup setup = new ServiceITSetupImpl(); + + @org.junit.Test public void testBasicOperation() throws Exception { @@@ -53,16 -48,12 +55,18 @@@ String orgName = "org_" + rand; String appName = "app_" + rand; + String userName = "user_" + rand; + + ExportDataCreator creator = new ExportDataCreator(); creator.startTool( new String[] { "-organization", orgName, "-application", appName, - "-host", "localhost:" + setup.getCassSvc().getCassandraHostConfigurator().getPort() + "-username", userName, + "-host", "localhost:9160", + "-eshost", "localhost:9200", + "-escluster", "elasticsearch" ++ }, false); long start = System.currentTimeMillis(); http://git-wip-us.apache.org/repos/asf/usergrid/blob/c4813b5f/stack/tools/src/test/java/org/apache/usergrid/tools/ExportImportAdminsTest.java ---------------------------------------------------------------------- diff --cc stack/tools/src/test/java/org/apache/usergrid/tools/ExportImportAdminsTest.java index 745ec9a,ceb6074..f0d599c --- a/stack/tools/src/test/java/org/apache/usergrid/tools/ExportImportAdminsTest.java +++ b/stack/tools/src/test/java/org/apache/usergrid/tools/ExportImportAdminsTest.java @@@ -48,16 -46,9 +49,16 @@@ import static junit.framework.TestCase. import static org.junit.Assert.*; - public class ExportImportAdminsTest { + public class ExportImportAdminsTest extends AbstractServiceIT { static final Logger logger = LoggerFactory.getLogger( ExportImportAdminsTest.class ); + @ClassRule + public static ServiceITSetup setup = new ServiceITSetupImpl(); + + public ExportImportAdminsTest() { + //logger.info("\n\n\nCREATING TEST -------------------------------------------\n\n\n"); + } + @org.junit.Test public void testExportUserAndOrg() throws Exception {
