Merge branch 'master' into datastax-cass-driver Fix issue with UniqueValueSerialization backwards compatibility via CQL and legacy Usergrid data.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/b23c20a2 Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/b23c20a2 Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/b23c20a2 Branch: refs/heads/master Commit: b23c20a28813522955f4c196e2fdd6e1638193ce Parents: 324f94d 91abfd8 Author: Michael Russo <[email protected]> Authored: Sun Jul 17 16:39:01 2016 -0700 Committer: Michael Russo <[email protected]> Committed: Sun Jul 17 16:39:01 2016 -0700 ---------------------------------------------------------------------- UsergridSDK.podspec | 18 + content/community/index.html | 3 +- content/index.html | 3 +- content/releases/index.html | 3 +- deployment/pcf/LICENSE | 201 ++ deployment/pcf/README.md | 47 +- deployment/pcf/addBlobs.sh | 25 + deployment/pcf/apache-usergrid-tile-1.6.yml | 510 ++++++ deployment/pcf/config/blobs.yml | 19 + deployment/pcf/config/final.yml | 24 + deployment/pcf/content_migrations.yml | 53 + deployment/pcf/createRelease.sh | 24 + deployment/pcf/createTile.sh | 32 + deployment/pcf/jobs/delete-all/monit | 17 + deployment/pcf/jobs/delete-all/spec | 44 + .../jobs/delete-all/templates/delete-all.sh.erb | 131 ++ deployment/pcf/jobs/deploy-all/monit | 18 + deployment/pcf/jobs/deploy-all/spec | 145 ++ .../jobs/deploy-all/templates/deploy-all.sh.erb | 384 ++++ .../pcf/jobs/docker-bosh-cassandra_docker/monit | 23 + .../pcf/jobs/docker-bosh-cassandra_docker/spec | 26 + .../docker-bosh-cassandra_docker.sh.erb | 69 + .../jobs/docker-bosh-elasticsearch_docker/monit | 23 + .../jobs/docker-bosh-elasticsearch_docker/spec | 26 + .../docker-bosh-elasticsearch_docker.sh.erb | 69 + .../pcf/packages/cassandra_docker/packaging | 26 + deployment/pcf/packages/cassandra_docker/spec | 26 + deployment/pcf/packages/cf_cli/packaging | 26 + deployment/pcf/packages/cf_cli/spec | 25 + deployment/pcf/packages/common/packaging | 23 + deployment/pcf/packages/common/spec | 25 + .../pcf/packages/elasticsearch_docker/packaging | 26 + .../pcf/packages/elasticsearch_docker/spec | 26 + deployment/pcf/packages/usergrid_app/packaging | 27 + deployment/pcf/packages/usergrid_app/spec | 27 + deployment/pcf/run.sh | 25 + deployment/pcf/src/common/utils.sh | 107 ++ deployment/pcf/src/templates/all_open.json | 6 + deployment/pcf/src/usergrid_app/manifest.yml | 22 + docs/data-storage/collections.md | 144 ++ docs/orgs-and-apps/application.md | 104 ++ sdks/java/README.md | 598 +++++- sdks/java/pom.xml | 73 +- .../org/apache/usergrid/java/client/Client.java | 1292 ------------- .../apache/usergrid/java/client/Usergrid.java | 285 +++ .../usergrid/java/client/UsergridClient.java | 427 +++++ .../java/client/UsergridClientConfig.java | 60 + .../usergrid/java/client/UsergridEnums.java | 170 ++ .../usergrid/java/client/UsergridRequest.java | 205 +++ .../java/client/UsergridRequestManager.java | 86 + .../java/client/auth/UsergridAppAuth.java | 55 + .../usergrid/java/client/auth/UsergridAuth.java | 75 + .../java/client/auth/UsergridUserAuth.java | 56 + .../usergrid/java/client/entities/Activity.java | 625 ------- .../usergrid/java/client/entities/Device.java | 68 - .../usergrid/java/client/entities/Entity.java | 191 -- .../usergrid/java/client/entities/Group.java | 79 - .../usergrid/java/client/entities/Message.java | 148 -- .../usergrid/java/client/entities/User.java | 158 -- .../java/client/exception/ClientException.java | 41 - .../client/exception/UsergridException.java | 50 + .../java/client/model/UsergridDevice.java | 60 + .../java/client/model/UsergridEntity.java | 487 +++++ .../java/client/model/UsergridUser.java | 198 ++ .../java/client/query/UsergridQuery.java | 434 +++++ .../java/client/response/AggregateCounter.java | 52 - .../client/response/AggregateCounterSet.java | 111 -- .../java/client/response/ApiResponse.java | 421 ----- .../client/response/ClientCredentialsInfo.java | 58 - .../java/client/response/QueueInfo.java | 44 - .../java/client/response/UsergridResponse.java | 230 +++ .../client/response/UsergridResponseError.java | 98 + .../usergrid/java/client/utils/JsonUtils.java | 262 ++- .../usergrid/java/client/utils/MapUtils.java | 27 +- .../usergrid/java/client/utils/ObjectUtils.java | 28 +- .../usergrid/java/client/utils/UrlUtils.java | 124 -- .../utils/UsergridEntityDeserializer.java | 41 + .../client/ClientAuthFallBackTestCase.java | 72 + .../usergrid/client/ClientAuthTestCase.java | 85 + .../client/ClientConnectionsTestCase.java | 171 ++ .../usergrid/client/ClientRestTestCase.java | 90 + .../apache/usergrid/client/EntityTestCase.java | 676 +++++++ .../apache/usergrid/client/QueryTestCase.java | 194 ++ .../usergrid/client/SDKTestConfiguration.java | 38 + .../apache/usergrid/client/SDKTestUtils.java | 108 ++ .../client/UsergridClientAuthTestCase.java | 73 + .../usergrid/client/UsergridInitTestCase.java | 48 + .../client/UsergridResponseErrorTestCase.java | 62 + .../client/UsergridResponseTestCase.java | 85 + .../usergrid/client/UsergridTestCase.java | 30 + sdks/java/usergrid-java-client-2.1.0.jar | Bin 0 -> 1992232 bytes sdks/swift/README.md | 2 +- .../Source/Base.lproj/Main.storyboard | 26 +- .../ActivityFeed/Source/FormTextField.swift | 2 +- .../Source/MessageViewController.swift | 26 +- .../Samples/Push/Source/UsergridManager.swift | 2 +- sdks/swift/Source/Usergrid.swift | 32 +- sdks/swift/Source/UsergridAsset.swift | 18 +- sdks/swift/Source/UsergridAuth.swift | 6 +- sdks/swift/Source/UsergridClient.swift | 89 +- sdks/swift/Source/UsergridClientConfig.swift | 14 +- sdks/swift/Source/UsergridDevice.swift | 41 +- sdks/swift/Source/UsergridEntity.swift | 104 +- sdks/swift/Source/UsergridEnums.swift | 22 +- sdks/swift/Source/UsergridExtensions.swift | 82 +- sdks/swift/Source/UsergridFileMetaData.swift | 4 +- sdks/swift/Source/UsergridQuery.swift | 3 +- sdks/swift/Source/UsergridRequest.swift | 2 +- sdks/swift/Source/UsergridRequestManager.swift | 72 +- sdks/swift/Source/UsergridResponse.swift | 6 +- sdks/swift/Source/UsergridUser.swift | 62 +- sdks/swift/Tests/ASSET_Tests.swift | 218 ++- sdks/swift/Tests/AUTH_Tests.swift | 92 +- sdks/swift/Tests/CONNECTION_Tests.swift | 42 +- sdks/swift/Tests/ClientCreationTests.swift | 44 +- sdks/swift/Tests/GET_Tests.swift | 38 +- sdks/swift/Tests/PUT_Tests.swift | 38 +- sdks/swift/Tests/TestAssets/UsergridGuy.jpg | Bin 0 -> 12981 bytes sdks/swift/Tests/User_Tests.swift | 310 +++- sdks/swift/UsergridSDK.podspec | 18 - .../swift/UsergridSDK.xcodeproj/project.pbxproj | 4 +- sdks/swift/docs/Classes.html | 16 +- sdks/swift/docs/Classes/Usergrid.html | 290 ++- sdks/swift/docs/Classes/UsergridAppAuth.html | 30 +- sdks/swift/docs/Classes/UsergridAsset.html | 44 +- .../Classes/UsergridAssetUploadRequest.html | 20 +- sdks/swift/docs/Classes/UsergridAuth.html | 34 +- sdks/swift/docs/Classes/UsergridClient.html | 275 ++- .../docs/Classes/UsergridClientConfig.html | 54 +- sdks/swift/docs/Classes/UsergridDevice.html | 205 ++- sdks/swift/docs/Classes/UsergridEntity.html | 215 ++- .../docs/Classes/UsergridFileMetaData.html | 30 +- sdks/swift/docs/Classes/UsergridQuery.html | 156 +- sdks/swift/docs/Classes/UsergridRequest.html | 36 +- sdks/swift/docs/Classes/UsergridResponse.html | 42 +- .../docs/Classes/UsergridResponseError.html | 24 +- sdks/swift/docs/Classes/UsergridUser.html | 231 ++- sdks/swift/docs/Classes/UsergridUserAuth.html | 30 +- sdks/swift/docs/Enums.html | 28 +- sdks/swift/docs/Enums/UsergridAuthMode.html | 283 +++ .../docs/Enums/UsergridDeviceProperties.html | 20 +- sdks/swift/docs/Enums/UsergridDirection.html | 16 +- .../docs/Enums/UsergridEntityProperties.html | 24 +- sdks/swift/docs/Enums/UsergridHttpMethod.html | 16 +- .../docs/Enums/UsergridImageContentType.html | 16 +- .../swift/docs/Enums/UsergridQueryOperator.html | 20 +- .../docs/Enums/UsergridQuerySortOrder.html | 20 +- .../docs/Enums/UsergridUserProperties.html | 20 +- sdks/swift/docs/Extensions.html | 72 +- sdks/swift/docs/Extensions/NSDate.html | 448 +++++ sdks/swift/docs/Global Variables.html | 18 +- sdks/swift/docs/Typealiases.html | 36 +- .../Contents/Resources/Documents/Classes.html | 209 ++- .../Resources/Documents/Classes/Usergrid.html | 1086 ++++++++--- .../Documents/Classes/UsergridAppAuth.html | 183 +- .../Documents/Classes/UsergridAsset.html | 265 ++- .../Classes/UsergridAssetUploadRequest.html | 356 ++++ .../Documents/Classes/UsergridAuth.html | 274 ++- .../Documents/Classes/UsergridClient.html | 1310 +++++++++---- .../Documents/Classes/UsergridClientConfig.html | 345 +++- .../Documents/Classes/UsergridDevice.html | 519 +++++- .../Documents/Classes/UsergridEntity.html | 809 +++++--- .../Documents/Classes/UsergridFileMetaData.html | 217 ++- .../Documents/Classes/UsergridQuery.html | 540 ++++-- .../Documents/Classes/UsergridRequest.html | 619 +++++++ .../Documents/Classes/UsergridResponse.html | 309 ++-- .../Classes/UsergridResponseError.html | 473 +++++ .../Documents/Classes/UsergridUser.html | 1734 ++++++++++++++++-- .../Documents/Classes/UsergridUserAuth.html | 169 +- .../Contents/Resources/Documents/Enums.html | 115 +- .../Documents/Enums/UsergridAuthFallback.html | 53 +- .../Documents/Enums/UsergridAuthMode.html | 283 +++ .../Enums/UsergridDeviceProperties.html | 83 +- .../Documents/Enums/UsergridDirection.html | 71 +- .../Enums/UsergridEntityProperties.html | 101 +- .../Documents/Enums/UsergridHttpMethod.html | 341 ++++ .../Enums/UsergridImageContentType.html | 73 +- .../Documents/Enums/UsergridQueryOperator.html | 89 +- .../Documents/Enums/UsergridQuerySortOrder.html | 83 +- .../Documents/Enums/UsergridUserProperties.html | 95 +- .../Resources/Documents/Extensions.html | 72 +- .../Resources/Documents/Extensions/NSDate.html | 448 +++++ .../Resources/Documents/Global Variables.html | 210 +++ .../Resources/Documents/Typealiases.html | 159 +- .../Resources/Documents/css/highlight.css | 6 +- .../Contents/Resources/Documents/css/jazzy.css | 65 +- .../Contents/Resources/Documents/index.html | 741 +++++++- .../Contents/Resources/Documents/js/jazzy.js | 11 +- .../Resources/Documents/undocumented.txt | 11 - .../.docset/Contents/Resources/docSet.dsidx | Bin 114688 -> 147456 bytes sdks/swift/docs/docsets/.tgz | Bin 111866 -> 148251 bytes sdks/swift/docs/index.html | 24 +- .../main/resources/usergrid-default.properties | 53 +- .../src/test/resources/usergrid-test.properties | 20 +- stack/core/pom.xml | 44 +- .../usergrid/corepersistence/CoreModule.java | 67 +- .../corepersistence/CpEntityManager.java | 407 ++-- .../corepersistence/CpEntityManagerFactory.java | 295 ++- .../corepersistence/CpRelationManager.java | 100 +- .../corepersistence/EntityManagerFig.java | 11 +- .../asyncevents/AsyncEventService.java | 6 +- .../asyncevents/AsyncEventServiceImpl.java | 45 +- .../asyncevents/EventBuilder.java | 6 +- .../asyncevents/EventBuilderImpl.java | 53 +- .../model/DeIndexOldVersionsEvent.java | 12 +- .../index/CollectionSettings.java | 47 + .../index/CollectionSettingsCache.java | 59 + .../index/CollectionSettingsCacheFig.java | 39 + .../index/CollectionSettingsFactory.java | 73 + .../index/CollectionSettingsImpl.java | 92 + .../index/CollectionSettingsScope.java | 27 + .../index/CollectionSettingsScopeImpl.java | 73 + .../corepersistence/index/IndexSchemaCache.java | 50 - .../index/IndexSchemaCacheFactory.java | 44 - .../index/IndexSchemaCacheFig.java | 39 - .../index/IndexSchemaCacheImpl.java | 120 -- .../corepersistence/index/IndexService.java | 23 +- .../corepersistence/index/IndexServiceImpl.java | 148 +- .../index/ReIndexServiceImpl.java | 41 +- .../pipeline/cursor/CursorSerializerUtil.java | 9 - .../pipeline/cursor/RequestCursor.java | 29 +- .../read/search/CandidateEntityFilter.java | 94 +- .../service/ApplicationServiceImpl.java | 27 +- .../apache/usergrid/locking/LockManager.java | 5 + .../locking/cassandra/AstyanaxLockImpl.java | 2 +- .../cassandra/AstyanaxLockManagerImpl.java | 113 +- .../locking/noop/NoOpLockManagerImpl.java | 5 + .../usergrid/persistence/EntityManager.java | 26 +- .../apache/usergrid/persistence/PathQuery.java | 3 +- .../usergrid/persistence/RelationManager.java | 2 + .../persistence/entities/Notification.java | 38 +- .../index/AsyncIndexServiceTest.java | 2 +- .../corepersistence/index/IndexServiceTest.java | 92 +- .../usergrid/corepersistence/index/RxTest.java | 11 + .../usergrid/persistence/CoreSchemaManager.java | 8 +- .../apache/usergrid/persistence/IndexIT.java | 98 + .../resources/usergrid-custom-test.properties | 13 + .../test/resources/usergrid-test-context.xml | 1 + stack/corepersistence/actorsystem/pom.xml | 106 ++ .../persistence/actorsystem/ActorSystemFig.java | 83 + .../actorsystem/ActorSystemManager.java | 86 + .../actorsystem/ActorSystemManagerImpl.java | 459 +++++ .../actorsystem/ActorSystemModule.java | 34 + .../persistence/actorsystem/ClientActor.java | 205 +++ .../actorsystem/GuiceActorProducer.java | 46 + .../persistence/actorsystem/RouterProducer.java | 51 + .../src/main/resources/application.conf | 50 + .../actorsystem/ActorServiceServiceTest.java | 74 + .../usergrid/persistence/cache/CacheScope.java | 24 + stack/corepersistence/collection/pom.xml | 120 +- .../collection/EntityCollectionManager.java | 21 +- .../EntityCollectionManagerFactory.java | 4 +- .../persistence/collection/FieldSet.java | 12 + .../exception/CollectionRuntimeException.java | 11 + .../exception/WriteUniqueVerifyException.java | 2 +- .../collection/guice/CollectionModule.java | 9 + .../EntityCollectionManagerFactoryImpl.java | 170 +- .../impl/EntityCollectionManagerImpl.java | 205 ++- .../mvcc/stage/CollectionIoEvent.java | 14 +- .../mvcc/stage/delete/MarkCommit.java | 36 +- .../mvcc/stage/write/WriteCommit.java | 87 +- .../mvcc/stage/write/WriteUniqueVerify.java | 155 +- .../serialization/SerializationFig.java | 1 + .../UniqueValueSerializationStrategy.java | 24 +- .../serialization/impl/LogEntryIterator.java | 128 ++ .../serialization/impl/MutableFieldSet.java | 12 + .../impl/UniqueFieldRowKeySerializer.java | 3 +- .../UniqueValueSerializationStrategyImpl.java | 267 ++- ...iqueValueSerializationStrategyProxyImpl.java | 20 +- .../UniqueValueSerializationStrategyV1Impl.java | 20 +- .../UniqueValueSerializationStrategyV2Impl.java | 22 +- .../uniquevalues/ReservationCache.java | 87 + .../uniquevalues/ReservationCacheActor.java | 87 + .../uniquevalues/UniqueValueActor.java | 282 +++ .../uniquevalues/UniqueValueException.java | 33 + .../uniquevalues/UniqueValuesFig.java | 67 + .../uniquevalues/UniqueValuesRouter.java | 70 + .../uniquevalues/UniqueValuesService.java | 74 + .../uniquevalues/UniqueValuesServiceImpl.java | 372 ++++ .../uniquevalues/UniqueValuesTable.java | 42 + .../uniquevalues/UniqueValuesTableImpl.java | 101 + .../collection/AbstractUniqueValueTest.java | 50 + .../collection/EntityCollectionManagerIT.java | 176 +- .../EntityCollectionManagerStressTest.java | 21 +- .../collection/guice/TestCollectionModule.java | 17 + .../mvcc/stage/delete/MarkCommitTest.java | 12 +- .../mvcc/stage/write/WriteCommitTest.java | 10 +- .../mvcc/stage/write/WriteUniqueVerifyIT.java | 125 +- .../mvcc/stage/write/WriteUniqueVerifyTest.java | 58 +- ...niqueValueSerializationStrategyImplTest.java | 229 ++- .../UniqueValuesServiceDeleteTest.java | 149 ++ .../uniquevalues/UniqueValuesServiceTest.java | 185 ++ .../src/test/resources/usergrid-CHOP.properties | 18 + .../src/test/resources/usergrid-UNIT.properties | 19 + .../src/test/resources/usergrid.properties | 32 + stack/corepersistence/common/pom.xml | 21 +- .../usergrid/persistence/core/CassandraFig.java | 18 +- .../core/astyanax/CassandraCluster.java | 24 +- .../core/guice/SettingsValidationCluster.java | 18 + .../core/guice/MigrationManagerRule.java | 18 + .../guice/SettingsValidationClusterTest.java | 19 + .../src/test/resources/usergrid-UNIT.properties | 19 + .../src/test/resources/usergrid.properties | 19 + .../graph/impl/GraphManagerImpl.java | 4 + .../impl/shard/DirectedEdgeMeta.java | 8 +- stack/corepersistence/model/pom.xml | 16 + .../persistence/model/entity/EntityMap.java | 18 + .../persistence/model/field/ArrayField.java | 7 + .../persistence/model/field/ByteArrayField.java | 9 + .../persistence/model/field/DistanceField.java | 9 + .../model/field/EntityObjectField.java | 9 + .../persistence/model/field/FieldTypeName.java | 3 - .../persistence/model/field/ListField.java | 7 + .../persistence/model/field/LocationField.java | 7 + .../persistence/model/field/NullField.java | 7 + .../persistence/model/field/SetField.java | 7 + .../model/field/value/EntityObject.java | 15 +- .../persistence/model/util/EntityUtils.java | 18 + .../persistence/model/util/UUIDGenerator.java | 18 + .../persistence/model/field/EntityTest.java | 18 + .../model/util/UUIDGeneratorTest.java | 18 + stack/corepersistence/pom.xml | 72 + stack/corepersistence/queryindex/pom.xml | 18 +- .../usergrid/persistence/index/EntityIndex.java | 5 +- .../usergrid/persistence/index/IndexFig.java | 2 +- .../index/impl/EsEntityIndexFactoryImpl.java | 2 + .../index/impl/EsEntityIndexImpl.java | 117 +- .../persistence/index/impl/EntityIndexTest.java | 41 - stack/pom.xml | 10 +- .../usergrid/rest/AbstractContextResource.java | 1 + .../apache/usergrid/rest/ShutdownListener.java | 11 + .../rest/applications/CollectionResource.java | 97 +- .../rest/applications/ServiceResource.java | 8 +- .../rest/applications/users/UserResource.java | 9 + .../rest/applications/users/UsersResource.java | 9 + ...ApplicationAlreadyExistsExceptionMapper.java | 32 + .../rest/management/ManagementResource.java | 2 +- .../organizations/OrganizationsResource.java | 18 +- .../rest/management/users/UserResource.java | 25 +- .../rest/management/users/UsersResource.java | 10 +- .../security/SecuredResourceFilterFactory.java | 86 +- .../apache/usergrid/rest/NotificationsIT.java | 6 +- .../org/apache/usergrid/rest/UniqueCatsIT.java | 233 +++ .../apache/usergrid/rest/UniqueValuesIT.java | 224 +++ .../rest/UniqueValuesPerformanceIT.java | 165 ++ .../rest/applications/ApplicationDeleteIT.java | 115 +- .../applications/ApplicationResourceIT.java | 7 +- .../usergrid/rest/applications/SecurityIT.java | 1 - .../collection/CollectionsResourceIT.java | 270 +-- .../collection/devices/DevicesResourceIT.java | 3 +- .../collection/users/PermissionsResourceIT.java | 129 +- .../queries/SelectMappingsQueryTest.java | 168 ++ .../usergrid/rest/management/AdminUsersIT.java | 80 + .../rest/management/ManagementResourceIT.java | 2 +- .../rest/management/OrganizationsIT.java | 3 - .../resource/endpoints/CollectionEndpoint.java | 24 +- .../test/resource/endpoints/NamedResource.java | 8 +- .../resource/endpoints/mgmt/OrgResource.java | 5 - .../resources/corepersistence-UNIT.properties | 2 + .../resources/usergrid-custom-test.properties | 19 +- .../resources/usergrid-rest-deploy-context.xml | 33 +- .../test/resources/usergrid-test-context.xml | 2 +- stack/services/pom.xml | 6 + .../cassandra/ManagementServiceImpl.java | 33 +- .../usergrid/security/shiro/ShiroCache.java | 70 +- .../principals/ApplicationUserPrincipal.java | 13 +- .../security/shiro/utils/LocalShiroCache.java | 2 +- .../services/AbstractCollectionService.java | 118 +- .../services/AbstractConnectionsService.java | 13 +- .../usergrid/services/AbstractService.java | 53 +- .../usergrid/services/ServiceManager.java | 8 + .../services/ServiceManagerFactory.java | 12 + .../applications/ApplicationsService.java | 4 +- .../services/devices/DevicesService.java | 2 + .../ApplicationQueueManagerCache.java | 2 +- .../impl/ApplicationQueueManagerImpl.java | 52 +- .../resources/usergrid-services-context.xml | 2 +- .../usergrid/services/ServiceInvocationIT.java | 28 +- .../AbstractServiceNotificationIT.java | 26 +- .../test/resources/usergrid-test-context.xml | 1 + stack/tools/pom.xml | 1 + .../java/org/apache/usergrid/tools/Cli.java | 2 +- .../org/apache/usergrid/tools/EntityUpdate.java | 3 +- .../org/apache/usergrid/tools/ExportApp.java | 2 +- .../java/org/apache/usergrid/tools/Import.java | 10 +- .../org/apache/usergrid/tools/ToolBase.java | 79 +- .../usergrid/tools/UniqueValueScanner.java | 298 +++ .../main/resources/toolsApplicationContext.xml | 7 + website/README.md | 2 +- website/layouts/footer.html | 3 +- 390 files changed, 30666 insertions(+), 8553 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/usergrid/blob/b23c20a2/stack/config/src/main/resources/usergrid-default.properties ---------------------------------------------------------------------- diff --cc stack/config/src/main/resources/usergrid-default.properties index 7c12783,95b3e9b..e356efc --- a/stack/config/src/main/resources/usergrid-default.properties +++ b/stack/config/src/main/resources/usergrid-default.properties @@@ -59,7 -59,7 +59,7 @@@ usergrid.persistence=C # Set a property to tell Usergrid which version of cassandra is being used. # - #cassandra.version=2.1 -#cassandra.version=1.2 ++cassandra.version=2.1 # Set the Cassandra cluster name that this instance of Usergrid should use. http://git-wip-us.apache.org/repos/asf/usergrid/blob/b23c20a2/stack/core/pom.xml ---------------------------------------------------------------------- diff --cc stack/core/pom.xml index 4b2252e,6c70342..9f2dc88 --- a/stack/core/pom.xml +++ b/stack/core/pom.xml @@@ -128,9 -130,9 +130,13 @@@ <groupId>org.antlr</groupId> </exclusion> <exclusion> + <groupId>net.jpountz.lz4</groupId> + <artifactId>*</artifactId> + </exclusion> ++ <exclusion> + <artifactId>netty</artifactId> + <groupId>io.netty</groupId> + </exclusion> </exclusions> </dependency> @@@ -362,9 -366,9 +370,7 @@@ <dependency> <groupId>org.apache.usergrid</groupId> <artifactId>common</artifactId> - <version>2.2.0-SNAPSHOT</version> - <type>test-jar</type> - <scope>test</scope> + <version>${project.version}</version> - <type>test-jar</type> - <scope>test</scope> </dependency> http://git-wip-us.apache.org/repos/asf/usergrid/blob/b23c20a2/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexServiceImpl.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/usergrid/blob/b23c20a2/stack/core/src/main/java/org/apache/usergrid/locking/cassandra/AstyanaxLockManagerImpl.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/usergrid/blob/b23c20a2/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/RxTest.java ---------------------------------------------------------------------- diff --cc stack/core/src/test/java/org/apache/usergrid/corepersistence/index/RxTest.java index f44c028,f44c028..f7d52d6 --- a/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/RxTest.java +++ b/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/RxTest.java @@@ -163,6 -163,6 +163,17 @@@ public class RxTest } ++ @Test ++ public void someTest(){ ++ ++ ++ final String uuidtype = "UUIDType"; ++ final String utf8type = "UTF8Type"; ++ ++ assertEquals(uuidtype.length(), utf8type.length()); ++ ++ } ++ private List<StreamResult> callStream (final List<Integer> input){ Stream<StreamResult> results = input.stream().map(integer -> { http://git-wip-us.apache.org/repos/asf/usergrid/blob/b23c20a2/stack/core/src/test/resources/usergrid-custom-test.properties ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/usergrid/blob/b23c20a2/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerFactoryImpl.java ---------------------------------------------------------------------- diff --cc stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerFactoryImpl.java index 71e56f5,fcaa51d..9f74927 --- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerFactoryImpl.java +++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerFactoryImpl.java @@@ -21,12 -19,16 +19,18 @@@ package org.apache.usergrid.persistence.collection.impl; - import java.util.concurrent.ExecutionException; + +import com.datastax.driver.core.Session; + import com.google.common.base.Preconditions; + import com.google.common.cache.CacheBuilder; + import com.google.common.cache.CacheLoader; + import com.google.common.cache.LoadingCache; + import com.google.inject.Inject; + import com.google.inject.Singleton; + import com.netflix.astyanax.Keyspace; + import org.apache.usergrid.persistence.actorsystem.ActorSystemManager; import org.apache.usergrid.persistence.collection.EntityCollectionManager; import org.apache.usergrid.persistence.collection.EntityCollectionManagerFactory; - import org.apache.usergrid.persistence.collection.cache.EntityCacheFig; import org.apache.usergrid.persistence.collection.mvcc.stage.delete.MarkCommit; import org.apache.usergrid.persistence.collection.mvcc.stage.delete.MarkStart; import org.apache.usergrid.persistence.collection.mvcc.stage.delete.UniqueCleanup; @@@ -41,6 -39,8 +41,8 @@@ import org.apache.usergrid.persistence. import org.apache.usergrid.persistence.collection.serialization.MvccLogEntrySerializationStrategy; import org.apache.usergrid.persistence.collection.serialization.SerializationFig; import org.apache.usergrid.persistence.collection.serialization.UniqueValueSerializationStrategy; + import org.apache.usergrid.persistence.collection.uniquevalues.UniqueValuesService; -import org.apache.usergrid.persistence.core.astyanax.CassandraConfig; ++import org.apache.usergrid.persistence.core.CassandraConfig; import org.apache.usergrid.persistence.core.metrics.MetricsFactory; import org.apache.usergrid.persistence.core.rx.RxTaskScheduler; import org.apache.usergrid.persistence.core.scope.ApplicationScope; @@@ -76,22 -69,42 +71,44 @@@ public class EntityCollectionManagerFac private final UniqueValueSerializationStrategy uniqueValueSerializationStrategy; private final MvccLogEntrySerializationStrategy mvccLogEntrySerializationStrategy; private final Keyspace keyspace; + private final Session session; private final MetricsFactory metricsFactory; private final RxTaskScheduler rxTaskScheduler; + private final ActorSystemManager actorSystemManager; + private final UniqueValuesService uniqueValuesService; + + private final CassandraConfig cassandraConfig; private LoadingCache<ApplicationScope, EntityCollectionManager> ecmCache = CacheBuilder.newBuilder().maximumSize( 1000 ) .build( new CacheLoader<ApplicationScope, EntityCollectionManager>() { public EntityCollectionManager load( ApplicationScope scope ) { - //create the target EM that will perform logic + //create the target EM that will perform logic final EntityCollectionManager target = new EntityCollectionManagerImpl( - writeStart, writeVerifyUnique, - writeOptimisticVerify, writeCommit, rollback, markStart, markCommit, uniqueCleanup, versionCompact, - entitySerializationStrategy, uniqueValueSerializationStrategy, - mvccLogEntrySerializationStrategy, keyspace, - metricsFactory, serializationFig, - rxTaskScheduler, scope, session ); + + writeStart, + writeVerifyUnique, + writeOptimisticVerify, + writeCommit, + rollback, + markStart, + markCommit, + uniqueCleanup, + versionCompact, + + entitySerializationStrategy, + uniqueValueSerializationStrategy, + mvccLogEntrySerializationStrategy, + + keyspace, + metricsFactory, + serializationFig, + rxTaskScheduler, + actorSystemManager, + uniqueValuesService, + cassandraConfig, - scope ); ++ scope, ++ session); return target; } @@@ -99,38 -112,49 +116,52 @@@ @Inject - public EntityCollectionManagerFactoryImpl( final WriteStart writeStart, final WriteUniqueVerify writeVerifyUnique, - final WriteOptimisticVerify writeOptimisticVerify, - final WriteCommit writeCommit, final RollbackAction rollback, - final MarkStart markStart, final MarkCommit markCommit, - final UniqueCleanup uniqueCleanup, final VersionCompact versionCompact, - final SerializationFig serializationFig, final - MvccEntitySerializationStrategy entitySerializationStrategy, - final UniqueValueSerializationStrategy uniqueValueSerializationStrategy, - final MvccLogEntrySerializationStrategy mvccLogEntrySerializationStrategy, - final Keyspace keyspace, final EntityCacheFig entityCacheFig, - final MetricsFactory metricsFactory, - @CollectionExecutorScheduler final RxTaskScheduler rxTaskScheduler, - final Session session ) { - - this.writeStart = writeStart; - this.writeVerifyUnique = writeVerifyUnique; - this.writeOptimisticVerify = writeOptimisticVerify; - this.writeCommit = writeCommit; - this.rollback = rollback; - this.markStart = markStart; - this.markCommit = markCommit; - this.uniqueCleanup = uniqueCleanup; - this.versionCompact = versionCompact; - this.serializationFig = serializationFig; - this.entitySerializationStrategy = entitySerializationStrategy; - this.uniqueValueSerializationStrategy = uniqueValueSerializationStrategy; + public EntityCollectionManagerFactoryImpl( + final WriteStart writeStart, + final WriteUniqueVerify writeVerifyUnique, + final WriteOptimisticVerify writeOptimisticVerify, + final WriteCommit writeCommit, + final RollbackAction rollback, + final MarkStart markStart, + final MarkCommit markCommit, + final UniqueCleanup uniqueCleanup, + final VersionCompact versionCompact, + final SerializationFig serializationFig, + final MvccEntitySerializationStrategy entitySerializationStrategy, + final UniqueValueSerializationStrategy uniqueValueSerializationStrategy, + final MvccLogEntrySerializationStrategy mvccLogEntrySerializationStrategy, + final Keyspace keyspace, + final MetricsFactory metricsFactory, + @CollectionExecutorScheduler + final RxTaskScheduler rxTaskScheduler, + final ActorSystemManager actorSystemManager, + final UniqueValuesService uniqueValuesService, - final CassandraConfig cassandraConfig ) { ++ final CassandraConfig cassandraConfig, ++ final Session session ) { + + this.writeStart = writeStart; + this.writeVerifyUnique = writeVerifyUnique; + this.writeOptimisticVerify = writeOptimisticVerify; + this.writeCommit = writeCommit; + this.rollback = rollback; + this.markStart = markStart; + this.markCommit = markCommit; + this.uniqueCleanup = uniqueCleanup; + this.versionCompact = versionCompact; + this.serializationFig = serializationFig; + this.entitySerializationStrategy = entitySerializationStrategy; + this.uniqueValueSerializationStrategy = uniqueValueSerializationStrategy; this.mvccLogEntrySerializationStrategy = mvccLogEntrySerializationStrategy; - this.keyspace = keyspace; - this.metricsFactory = metricsFactory; - this.rxTaskScheduler = rxTaskScheduler; + this.keyspace = keyspace; + this.metricsFactory = metricsFactory; + this.rxTaskScheduler = rxTaskScheduler; + this.actorSystemManager = actorSystemManager; + this.uniqueValuesService = uniqueValuesService; + this.cassandraConfig = cassandraConfig; + this.session = session; ++ } + @Override public EntityCollectionManager createCollectionManager(ApplicationScope applicationScope) { Preconditions.checkNotNull(applicationScope); http://git-wip-us.apache.org/repos/asf/usergrid/blob/b23c20a2/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java ---------------------------------------------------------------------- diff --cc stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java index 291e5df,9dce7ef..a61e744 --- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java +++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java @@@ -19,43 -19,32 +19,35 @@@ package org.apache.usergrid.persistence.collection.impl; - import java.util.ArrayList; - import java.util.Collection; - import java.util.Collections; - import java.util.Iterator; - import java.util.List; - import java.util.UUID; + +import com.datastax.driver.core.BatchStatement; +import com.datastax.driver.core.Session; + import com.codahale.metrics.Timer; + import com.google.common.base.Preconditions; + import com.google.inject.Inject; + import com.google.inject.assistedinject.Assisted; + import com.netflix.astyanax.Keyspace; + import com.netflix.astyanax.MutationBatch; + import com.netflix.astyanax.connectionpool.OperationResult; + import com.netflix.astyanax.connectionpool.exceptions.ConnectionException; + import com.netflix.astyanax.model.ColumnFamily; import com.netflix.astyanax.model.ConsistencyLevel; - import org.slf4j.Logger; - import org.slf4j.LoggerFactory; - - import org.apache.usergrid.persistence.collection.EntityCollectionManager; - import org.apache.usergrid.persistence.collection.EntitySet; - import org.apache.usergrid.persistence.collection.FieldSet; - import org.apache.usergrid.persistence.collection.MvccEntity; - import org.apache.usergrid.persistence.collection.MvccLogEntry; - import org.apache.usergrid.persistence.collection.VersionSet; + import com.netflix.astyanax.model.CqlResult; + import com.netflix.astyanax.serializers.StringSerializer; + import org.apache.usergrid.persistence.actorsystem.ActorSystemManager; + import org.apache.usergrid.persistence.collection.*; import org.apache.usergrid.persistence.collection.mvcc.stage.CollectionIoEvent; import org.apache.usergrid.persistence.collection.mvcc.stage.delete.MarkCommit; import org.apache.usergrid.persistence.collection.mvcc.stage.delete.MarkStart; import org.apache.usergrid.persistence.collection.mvcc.stage.delete.UniqueCleanup; import org.apache.usergrid.persistence.collection.mvcc.stage.delete.VersionCompact; - import org.apache.usergrid.persistence.collection.mvcc.stage.write.RollbackAction; - import org.apache.usergrid.persistence.collection.mvcc.stage.write.WriteCommit; - import org.apache.usergrid.persistence.collection.mvcc.stage.write.WriteOptimisticVerify; - import org.apache.usergrid.persistence.collection.mvcc.stage.write.WriteStart; - import org.apache.usergrid.persistence.collection.mvcc.stage.write.WriteUniqueVerify; - import org.apache.usergrid.persistence.collection.serialization.MvccEntitySerializationStrategy; - import org.apache.usergrid.persistence.collection.serialization.MvccLogEntrySerializationStrategy; - import org.apache.usergrid.persistence.collection.serialization.SerializationFig; - import org.apache.usergrid.persistence.collection.serialization.UniqueValue; - import org.apache.usergrid.persistence.collection.serialization.UniqueValueSerializationStrategy; - import org.apache.usergrid.persistence.collection.serialization.UniqueValueSet; + import org.apache.usergrid.persistence.collection.mvcc.stage.write.*; + import org.apache.usergrid.persistence.collection.serialization.*; + import org.apache.usergrid.persistence.collection.serialization.impl.LogEntryIterator; import org.apache.usergrid.persistence.collection.serialization.impl.MinMaxLogEntryIterator; import org.apache.usergrid.persistence.collection.serialization.impl.MutableFieldSet; + import org.apache.usergrid.persistence.collection.uniquevalues.UniqueValuesService; -import org.apache.usergrid.persistence.core.astyanax.CassandraConfig; ++import org.apache.usergrid.persistence.core.CassandraConfig; import org.apache.usergrid.persistence.core.metrics.MetricsFactory; import org.apache.usergrid.persistence.core.metrics.ObservableTimer; import org.apache.usergrid.persistence.core.rx.ObservableIterator; @@@ -67,20 -56,12 +59,12 @@@ import org.apache.usergrid.persistence. import org.apache.usergrid.persistence.model.entity.Id; import org.apache.usergrid.persistence.model.field.Field; import org.apache.usergrid.persistence.model.util.UUIDGenerator; - - import com.codahale.metrics.Timer; - import com.google.common.base.Preconditions; - import com.google.inject.Inject; - import com.google.inject.assistedinject.Assisted; - import com.netflix.astyanax.Keyspace; - import com.netflix.astyanax.connectionpool.OperationResult; - import com.netflix.astyanax.connectionpool.exceptions.ConnectionException; - import com.netflix.astyanax.model.ColumnFamily; - import com.netflix.astyanax.model.CqlResult; - import com.netflix.astyanax.serializers.StringSerializer; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import rx.Observable; import rx.Subscriber; ++import org.slf4j.Logger; ++import org.slf4j.LoggerFactory; + + import java.util.*; /** @@@ -124,20 -105,35 +109,36 @@@ public class EntityCollectionManagerImp private final ApplicationScope applicationScope; private final RxTaskScheduler rxTaskScheduler; + private final UniqueValuesService uniqueValuesService; + private final ActorSystemManager actorSystemManager; + @Inject - public EntityCollectionManagerImpl( final WriteStart writeStart, final WriteUniqueVerify writeVerifyUnique, - final WriteOptimisticVerify writeOptimisticVerify, - final WriteCommit writeCommit, final RollbackAction rollback, - final MarkStart markStart, final MarkCommit markCommit, - final UniqueCleanup uniqueCleanup, final VersionCompact versionCompact, - final MvccEntitySerializationStrategy entitySerializationStrategy, - final UniqueValueSerializationStrategy uniqueValueSerializationStrategy, - final MvccLogEntrySerializationStrategy mvccLogEntrySerializationStrategy, - final Keyspace keyspace, final MetricsFactory metricsFactory, - final SerializationFig serializationFig, final RxTaskScheduler rxTaskScheduler, - @Assisted final ApplicationScope applicationScope, - final Session session) { + public EntityCollectionManagerImpl( + final WriteStart writeStart, + final WriteUniqueVerify writeVerifyUnique, + final WriteOptimisticVerify writeOptimisticVerify, + final WriteCommit writeCommit, + final RollbackAction rollback, + final MarkStart markStart, + final MarkCommit markCommit, + final UniqueCleanup uniqueCleanup, + final VersionCompact versionCompact, + + final MvccEntitySerializationStrategy entitySerializationStrategy, + final UniqueValueSerializationStrategy uniqueValueSerializationStrategy, + final MvccLogEntrySerializationStrategy mvccLogEntrySerializationStrategy, + + final Keyspace keyspace, + final MetricsFactory metricsFactory, + final SerializationFig serializationFig, + final RxTaskScheduler rxTaskScheduler, + final ActorSystemManager actorSystemManager, + final UniqueValuesService uniqueValuesService, + final CassandraConfig cassandraConfig, - @Assisted final ApplicationScope applicationScope ) { ++ @Assisted final ApplicationScope applicationScope, ++ final Session session ) { + this.uniqueValueSerializationStrategy = uniqueValueSerializationStrategy; this.entitySerializationStrategy = entitySerializationStrategy; this.uniqueCleanup = uniqueCleanup; @@@ -311,65 -328,105 +330,106 @@@ * Retrieves all entities that correspond to each field given in the Collection. */ @Override - public Observable<FieldSet> getEntitiesFromFields( final String type, final Collection<Field> fields ) { + public Observable<FieldSet> getEntitiesFromFields(final String type, final Collection<Field> fields, + boolean uniqueIndexRepair) { final Observable<FieldSet> fieldSetObservable = Observable.just( fields ).map( fields1 -> { - try { - final UUID startTime = UUIDGenerator.newTimeUUID(); + final UUID startTime = UUIDGenerator.newTimeUUID(); - //Get back set of unique values that correspond to collection of fields - UniqueValueSet set = uniqueValueSerializationStrategy.load( applicationScope, type, fields1 ); + //Get back set of unique values that correspond to collection of fields + //Purposely use string consistency as it's extremely important here, regardless of performance + UniqueValueSet set = + uniqueValueSerializationStrategy - .load( applicationScope, cassandraConfig.getConsistentReadCL(), type, fields1 , uniqueIndexRepair); ++ .load( applicationScope, cassandraConfig.getDataStaxReadConsistentCl(), type, fields1 , uniqueIndexRepair); + + //Short circuit if we don't have any uniqueValues from the given fields. + if ( !set.iterator().hasNext() ) { + + fields1.forEach( field -> { + + if(logger.isTraceEnabled()){ + logger.trace("Requested field [{}={}] not found in unique value table", + field.getName(), field.getValue().toString()); + } + + }); + + if(logger.isTraceEnabled()) { + logger.trace("No unique values found for requested fields, returning empty FieldSet"); + } + + return new MutableFieldSet( 0 ); + } + //Short circuit if we don't have any uniqueValues from the given fields. + if ( !set.iterator().hasNext() ) { + return new MutableFieldSet( 0 ); + } - //loop through each field, and construct an entity load - List<Id> entityIds = new ArrayList<>( fields1.size() ); - List<UniqueValue> uniqueValues = new ArrayList<>( fields1.size() ); - for ( final Field expectedField : fields1 ) { + //loop through each field, and construct an entity load + List<Id> entityIds = new ArrayList<>( fields1.size() ); + List<UniqueValue> uniqueValues = new ArrayList<>( fields1.size() ); - UniqueValue value = set.getValue( expectedField.getName() ); + for ( final Field expectedField : fields1 ) { - if ( value == null ) { - logger.debug( "Field does not correspond to a unique value" ); - } + UniqueValue value = set.getValue( expectedField.getName() ); - entityIds.add( value.getEntityId() ); - uniqueValues.add( value ); + if ( value == null ) { + logger.debug( "Field does not correspond to a unique value" ); } - //Load a entity for each entityId we retrieved. - final EntitySet entitySet = entitySerializationStrategy.load( applicationScope, entityIds, startTime ); + entityIds.add( value.getEntityId() ); + uniqueValues.add( value ); + } - //now loop through and ensure the entities are there. - final MutationBatch deleteBatch = keyspace.prepareMutationBatch(); + //Load a entity for each entityId we retrieved. + final EntitySet entitySet = entitySerializationStrategy.load( applicationScope, entityIds, startTime ); - final MutableFieldSet response = new MutableFieldSet( fields1.size() ); + final BatchStatement uniqueDeleteBatch = new BatchStatement(); - for ( final UniqueValue expectedUnique : uniqueValues ) { - final MvccEntity entity = entitySet.getEntity( expectedUnique.getEntityId() ); + - //bad unique value, delete this, it's inconsistent - if ( entity == null || !entity.getEntity().isPresent() ) { + final MutableFieldSet response = new MutableFieldSet( fields1.size() ); - if(logger.isTraceEnabled()) { - logger.trace("Unique value [{}={}] does not have corresponding entity [{}], executing " + - "read repair to remove stale unique value entry", - expectedUnique.getField().getName(), - expectedUnique.getField().getValue().toString(), - expectedUnique.getEntityId() - ); - } + for ( final UniqueValue expectedUnique : uniqueValues ) { + final MvccEntity entity = entitySet.getEntity( expectedUnique.getEntityId() ); - final MutationBatch valueDelete = - uniqueValueSerializationStrategy.delete( applicationScope, expectedUnique ); + //bad unique value, delete this, it's inconsistent + if ( entity == null || !entity.getEntity().isPresent() ) { + - deleteBatch.mergeShallow( valueDelete ); - continue; ++ if(logger.isTraceEnabled()) { ++ logger.trace("Unique value [{}={}] does not have corresponding entity [{}], executing " + ++ "read repair to remove stale unique value entry", ++ expectedUnique.getField().getName(), ++ expectedUnique.getField().getValue().toString(), ++ expectedUnique.getEntityId() ++ ); + } + - //TODO, we need to validate the property in the entity matches the property in the unique value - - - //else add it to our result set - response.addEntity( expectedUnique.getField(), entity ); + uniqueDeleteBatch.add( + uniqueValueSerializationStrategy.deleteCQL( applicationScope, expectedUnique )); + continue; } - if ( deleteBatch.getRowCount() > 0 ) { + //TODO, we need to validate the property in the entity matches the property in the unique value - response.setEntityRepairExecuted(true); - deleteBatch.execute(); - } + - return response; + //else add it to our result set + response.addEntity( expectedUnique.getField(), entity ); } - catch ( ConnectionException e ) { - logger.error( "Failed to getIdField", e ); - throw new RuntimeException( e ); + - //TODO: explore making this an Async process - session.execute(uniqueDeleteBatch); ++ ++ if ( uniqueDeleteBatch.getStatements().size() > 0 ) { ++ ++ response.setEntityRepairExecuted(true); ++ //TODO: explore making this an Async process ++ session.execute(uniqueDeleteBatch); + } ++ + + return response; + } ); http://git-wip-us.apache.org/repos/asf/usergrid/blob/b23c20a2/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteCommit.java ---------------------------------------------------------------------- diff --cc stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteCommit.java index cfac8e4,5b98ca5..fc1382f --- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteCommit.java +++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteCommit.java @@@ -18,10 -18,15 +18,20 @@@ package org.apache.usergrid.persistence.collection.mvcc.stage.write; + import java.util.HashMap; + import java.util.Map; import java.util.UUID; ++ +import com.datastax.driver.core.BatchStatement; +import com.datastax.driver.core.Session; ++ + import org.apache.usergrid.persistence.actorsystem.ActorSystemFig; + import org.apache.usergrid.persistence.collection.exception.WriteUniqueVerifyException; + import org.apache.usergrid.persistence.collection.uniquevalues.UniqueValueException; + import org.apache.usergrid.persistence.collection.uniquevalues.UniqueValuesFig; + import org.apache.usergrid.persistence.collection.uniquevalues.UniqueValuesService; ++ import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@@ -76,7 -83,9 +90,11 @@@ public class WriteCommit implements Fun public WriteCommit( final MvccLogEntrySerializationStrategy logStrat, final MvccEntitySerializationStrategy entryStrat, final UniqueValueSerializationStrategy uniqueValueStrat, - final Session session) { + final ActorSystemFig actorSystemFig, + final UniqueValuesFig uniqueValuesFig, - final UniqueValuesService akkaUvService ) { ++ final UniqueValuesService akkaUvService, ++ final Session session ) { ++ Preconditions.checkNotNull( logStrat, "MvccLogEntrySerializationStrategy is required" ); Preconditions.checkNotNull( entryStrat, "MvccEntitySerializationStrategy is required" ); @@@ -85,7 -94,9 +103,11 @@@ this.logEntryStrat = logStrat; this.entityStrat = entryStrat; this.uniqueValueStrat = uniqueValueStrat; + this.actorSystemFig = actorSystemFig; + this.uniqueValuesFig = uniqueValuesFig; + this.akkaUvService = akkaUvService; + this.session = session; ++ } @@@ -107,10 -122,9 +133,11 @@@ MvccValidationUtils.verifyMvccEntityWithEntity( ioEvent.getEvent() ); ValidationUtils.verifyTimeUuid( version ,"version" ); - final MvccLogEntry startEntry = new MvccLogEntryImpl( entityId, version, Stage.COMMITTED, MvccLogEntry.State.COMPLETE ); + final MvccLogEntry startEntry = + new MvccLogEntryImpl( entityId, version, Stage.COMMITTED, MvccLogEntry.State.COMPLETE ); + + MutationBatch logMutation = logEntryStrat.write( applicationScope, startEntry ); // now get our actual insert into the entity data @@@ -119,15 -133,44 +146,46 @@@ // merge the 2 into 1 mutation logMutation.mergeShallow( entityMutation ); + // akkaFig may be null when this is called from JUnit tests + if ( actorSystemFig != null && actorSystemFig.getEnabled() ) { + String region = ioEvent.getRegion(); + if ( region == null ) { + region = uniqueValuesFig.getAuthoritativeRegion(); + } + if ( region == null ) { + region = actorSystemFig.getRegionLocal(); + } + confirmUniqueFieldsAkka( mvccEntity, version, applicationScope, region ); + } else { + confirmUniqueFields( mvccEntity, version, applicationScope, logMutation ); + } + + try { + logMutation.execute(); + } + catch ( ConnectionException e ) { + logger.error( "Failed to execute write asynchronously ", e ); + throw new WriteCommitException( mvccEntity, applicationScope, + "Failed to execute write asynchronously ", e ); + } + + return ioEvent; + } + + + private void confirmUniqueFields( + MvccEntity mvccEntity, UUID version, ApplicationScope scope, MutationBatch logMutation) { + + final Entity entity = mvccEntity.getEntity().get(); + // re-write the unique values but this time with no TTL + final BatchStatement uniqueBatch = new BatchStatement(); + for ( Field field : EntityUtils.getUniqueFields(mvccEntity.getEntity().get()) ) { - UniqueValue written = new UniqueValueImpl( field, - entityId,version); + UniqueValue written = new UniqueValueImpl( field, entity.getId(), version); - uniqueBatch.add(uniqueValueStrat.writeCQL(applicationScope, written, -1 )); - MutationBatch mb = uniqueValueStrat.write(scope, written ); ++ uniqueBatch.add(uniqueValueStrat.writeCQL(scope, written, -1 )); logger.debug("Finalizing {} unique value {}", field.getName(), field.getValue().toString()); http://git-wip-us.apache.org/repos/asf/usergrid/blob/b23c20a2/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java ---------------------------------------------------------------------- diff --cc stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java index 5e99a05,a3565ea..e7dbb10 --- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java +++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java @@@ -18,18 -18,18 +18,22 @@@ package org.apache.usergrid.persistence.collection.mvcc.stage.write; - import java.util.ArrayList; - import java.util.HashMap; - import java.util.List; - import java.util.Map; + +import com.datastax.driver.core.BatchStatement; +import com.datastax.driver.core.ConsistencyLevel; +import com.datastax.driver.core.Session; +import com.netflix.hystrix.HystrixCommandProperties; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + + import com.google.common.base.Preconditions; + import com.google.inject.Inject; + import com.google.inject.Singleton; + import com.netflix.astyanax.Keyspace; -import com.netflix.astyanax.MutationBatch; -import com.netflix.astyanax.connectionpool.exceptions.ConnectionException; -import com.netflix.astyanax.model.ConsistencyLevel; + import com.netflix.hystrix.HystrixCommand; + import com.netflix.hystrix.HystrixCommandGroupKey; -import com.netflix.hystrix.HystrixCommandProperties; + import com.netflix.hystrix.HystrixThreadPoolProperties; + import org.apache.usergrid.persistence.actorsystem.ActorSystemFig; import org.apache.usergrid.persistence.collection.MvccEntity; import org.apache.usergrid.persistence.collection.exception.WriteUniqueVerifyException; import org.apache.usergrid.persistence.collection.mvcc.entity.MvccValidationUtils; @@@ -39,24 -39,21 +43,20 @@@ import org.apache.usergrid.persistence. import org.apache.usergrid.persistence.collection.serialization.UniqueValueSerializationStrategy; import org.apache.usergrid.persistence.collection.serialization.UniqueValueSet; import org.apache.usergrid.persistence.collection.serialization.impl.UniqueValueImpl; +import org.apache.usergrid.persistence.core.CassandraConfig; + import org.apache.usergrid.persistence.collection.uniquevalues.UniqueValueException; + import org.apache.usergrid.persistence.collection.uniquevalues.UniqueValuesFig; + import org.apache.usergrid.persistence.collection.uniquevalues.UniqueValuesService; -import org.apache.usergrid.persistence.core.astyanax.CassandraConfig; import org.apache.usergrid.persistence.core.scope.ApplicationScope; import org.apache.usergrid.persistence.model.entity.Entity; import org.apache.usergrid.persistence.model.entity.Id; import org.apache.usergrid.persistence.model.field.Field; import org.apache.usergrid.persistence.model.util.EntityUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; + - import com.google.common.base.Preconditions; - import com.google.inject.Inject; - import com.google.inject.Singleton; - import com.netflix.astyanax.Keyspace; - import com.netflix.astyanax.connectionpool.exceptions.ConnectionException; - import com.netflix.hystrix.HystrixCommand; - import com.netflix.hystrix.HystrixCommandGroupKey; - import com.netflix.hystrix.HystrixThreadPoolProperties; - import rx.functions.Action1; + import java.util.*; + /** * This phase execute all unique value verification on the MvccEntity. @@@ -66,9 -63,13 +66,13 @@@ public class WriteUniqueVerify implemen private static final Logger logger = LoggerFactory.getLogger( WriteUniqueVerify.class ); - ActorSystemFig actorSystemFig; - UniqueValuesFig uniqueValuesFig; - UniqueValuesService akkaUvService; ++ private ActorSystemFig actorSystemFig; ++ private UniqueValuesFig uniqueValuesFig; ++ private UniqueValuesService akkaUvService; + private final UniqueValueSerializationStrategy uniqueValueStrat; - public static int uniqueVerifyPoolSize = 100; + private static int uniqueVerifyPoolSize = 100; private static int uniqueVerifyTimeoutMillis= 5000; @@@ -82,13 -80,19 +86,21 @@@ @Inject - public WriteUniqueVerify( final UniqueValueSerializationStrategy uniqueValueSerializiationStrategy, - final SerializationFig serializationFig, final Keyspace keyspace, - final CassandraConfig cassandraFig, final Session session ) { + public WriteUniqueVerify(final UniqueValueSerializationStrategy uniqueValueSerializiationStrategy, + final SerializationFig serializationFig, + final Keyspace keyspace, + final CassandraConfig cassandraFig, + final ActorSystemFig actorSystemFig, + final UniqueValuesFig uniqueValuesFig, - final UniqueValuesService akkaUvService ) { ++ final UniqueValuesService akkaUvService, ++ final Session session ) { this.keyspace = keyspace; this.cassandraFig = cassandraFig; + this.actorSystemFig = actorSystemFig; + this.uniqueValuesFig = uniqueValuesFig; + this.akkaUvService = akkaUvService; + this.session = session; Preconditions.checkNotNull( uniqueValueSerializiationStrategy, "uniqueValueSerializationStrategy is required" ); Preconditions.checkNotNull( serializationFig, "serializationFig is required" ); @@@ -128,10 -168,58 +176,52 @@@ // use write-first then read strategy final UniqueValue written = new UniqueValueImpl( field, mvccEntity.getId(), mvccEntity.getVersion() ); - // use TTL in case something goes wrong before entity is finally committed - batch.add(uniqueValueStrat.writeCQL( scope, written, serializationFig.getTimeout() )); - try { - uniqueFields.add(field); - // don't use read repair on this pre-write check - // stronger consistency is extremely important here, more so than performance - UniqueValueSet set = uniqueValueStrat.load(scope, cassandraFig.getConsistentReadCL(), - written.getEntityId().getType(), Collections.singletonList(written.getField()), false); ++ // don't use read repair on this pre-write check ++ // stronger consistency is extremely important here, more so than performance ++ UniqueValueSet set = uniqueValueStrat.load(scope, cassandraFig.getDataStaxReadConsistentCl(), ++ written.getEntityId().getType(), Collections.singletonList(written.getField()), false); + - set.forEach(uniqueValue -> { ++ set.forEach(uniqueValue -> { + - if(!uniqueValue.getEntityId().getUuid().equals(written.getEntityId().getUuid())){ - - if(logger.isTraceEnabled()){ - logger.trace("Pre-write violation detected. Attempted write for unique value [{}={}] and " + - "entity id [{}], entity version [{}] conflicts with already existing entity id [{}], " + - "entity version [{}]", - written.getField().getName(), - written.getField().getValue().toString(), - written.getEntityId().getUuid(), - written.getEntityVersion(), - uniqueValue.getEntityId().getUuid(), - uniqueValue.getEntityVersion()); - } - - preWriteUniquenessViolations.put(field.getName(), field); ++ if(!uniqueValue.getEntityId().getUuid().equals(written.getEntityId().getUuid())){ + ++ if(logger.isTraceEnabled()){ ++ logger.trace("Pre-write violation detected. Attempted write for unique value [{}={}] and " + ++ "entity id [{}], entity version [{}] conflicts with already existing entity id [{}], " + ++ "entity version [{}]", ++ written.getField().getName(), ++ written.getField().getValue().toString(), ++ written.getEntityId().getUuid(), ++ written.getEntityVersion(), ++ uniqueValue.getEntityId().getUuid(), ++ uniqueValue.getEntityVersion()); + } + - }); ++ preWriteUniquenessViolations.put(field.getName(), field); + ++ } ++ ++ }); + - } catch (ConnectionException e) { + - throw new RuntimeException("Error connecting to cassandra", e); - } + + // only build the batch statement if we don't have a violation for the field + if( preWriteUniquenessViolations.get(field.getName()) == null) { + + // use TTL in case something goes wrong before entity is finally committed - final MutationBatch mb = uniqueValueStrat.write(scope, written, serializationFig.getTimeout()); ++ batch.add(uniqueValueStrat.writeCQL(scope, written, serializationFig.getTimeout())); + - batch.mergeShallow(mb); + uniqueFields.add(field); + } + } + + if(preWriteUniquenessViolations.size() > 0 ){ + if(logger.isTraceEnabled()){ + logger.trace("Pre-write unique violations found, raising exception before executing first write"); + } + + throw new WriteUniqueVerifyException(mvccEntity, scope, preWriteUniquenessViolations ); } //short circuit nothing to do @@@ -140,13 -228,18 +230,14 @@@ } //perform the write - try { - batch.execute(); - } - catch ( ConnectionException ex ) { - throw new RuntimeException( "Unable to write to cassandra", ex ); - } + session.execute(batch); + // use simple thread pool to verify fields in parallel - ConsistentReplayCommand cmd = new ConsistentReplayCommand(uniqueValueStrat,cassandraFig,scope, entity.getId().getType(), uniqueFields,entity); + ConsistentReplayCommand cmd = new ConsistentReplayCommand( + uniqueValueStrat,cassandraFig,scope, entity.getId().getType(), uniqueFields,entity); - Map<String,Field> uniquenessViolations = cmd.execute(); + Map<String,Field> uniquenessViolations = cmd.execute(); //do we want to do this? @@@ -184,16 -278,24 +276,21 @@@ @Override protected Map<String, Field> getFallback() { - // fallback with same CL as there are many reasons the 1st execution failed, - // not just due to consistency problems - return executeStrategy(fig.getReadCL()); + // fallback with same CL as there are many reasons the 1st execution failed, not just due to consistency problems + return executeStrategy(fig.getDataStaxReadCl()); ++ } public Map<String, Field> executeStrategy(ConsistencyLevel consistencyLevel){ -- //allocate our max size, worst case -- //now get the set of fields back ++ final UniqueValueSet uniqueValues; - try { - // load ascending for verification to make sure we wrote is the last read back - // don't read repair on this read because our write-first strategy will introduce a duplicate - uniqueValues = - uniqueValueSerializationStrategy.load( scope, consistencyLevel, type, uniqueFields, false); - } - catch ( ConnectionException e ) { - throw new RuntimeException( "Unable to read from cassandra", e ); - } + - uniqueValues = uniqueValueSerializationStrategy.load( scope, consistencyLevel, type, uniqueFields ); ++ // load ascending for verification to make sure we wrote is the last read back ++ // don't read repair on this read because our write-first strategy will introduce a duplicate ++ uniqueValues = ++ uniqueValueSerializationStrategy.load( scope, consistencyLevel, type, uniqueFields, false); ++ ++ final Map<String, Field> uniquenessViolations = new HashMap<>( uniqueFields.size() ); http://git-wip-us.apache.org/repos/asf/usergrid/blob/b23c20a2/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/UniqueValueSerializationStrategy.java ---------------------------------------------------------------------- diff --cc stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/UniqueValueSerializationStrategy.java index eb43985,c6c70b9..cb6cd2b --- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/UniqueValueSerializationStrategy.java +++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/UniqueValueSerializationStrategy.java @@@ -43,12 -54,13 +43,13 @@@ public interface UniqueValueSerializati * @param applicationScope scope * @param uniqueValue Object to be written * @param timeToLive How long object should live in seconds. -1 implies store forever - * @return MutatationBatch that encapsulates operation, caller may or may not execute. + * @return BatchStatement that encapsulates CQL statements, caller may or may not execute. */ - MutationBatch write( ApplicationScope applicationScope, UniqueValue uniqueValue, int timeToLive ); + BatchStatement writeCQL(ApplicationScope applicationScope, UniqueValue uniqueValue, int timeToLive ); /** - * Load UniqueValue that matches field from collection or null if that value does not exist. + * Load UniqueValue that matches field from collection or null if that value does not exist. Returns the oldest + * unique value entry if more than 1 exists * * @param applicationScope scope in which to look for field name/value * @param type The type the unique value exists within @@@ -56,9 -68,26 +57,24 @@@ * * @return UniqueValueSet containing fields from the collection that exist in cassandra * - * @throws ConnectionException on error connecting to Cassandra */ - UniqueValueSet load( ApplicationScope applicationScope, String type, Collection<Field> fields ) - throws ConnectionException; + UniqueValueSet load( ApplicationScope applicationScope, String type, Collection<Field> fields ); + + + /** + * Load UniqueValue that matches field from collection or null if that value does not exist. Returns the oldest + * unique value entry if more than 1 exists + * + * @param applicationScope scope in which to look for field name/value + * @param type The type the unique value exists within + * @param fields Field name/value to search for + * + * @return UniqueValueSet containing fields from the collection that exist in cassandra + * - * @throws ConnectionException on error connecting to Cassandra + */ + UniqueValueSet load( ApplicationScope applicationScope, String type, Collection<Field> fields, - boolean useReadRepair ) throws ConnectionException; ++ boolean useReadRepair ); + /** * Load UniqueValue that matches field from collection or null if that value does not exist. * @@@ -66,11 -95,13 +82,13 @@@ * @param consistencyLevel Consistency level of query * @param type The type the unique value exists within * @param fields Field name/value to search for + * @return UniqueValueSet containing fields from the collection that exist in cassandra ++ * + * @param useReadRepair + * @return UniqueValueSet containing fields from the collection that exist in cassandra - * @throws ConnectionException on error connecting to Cassandra */ UniqueValueSet load(ApplicationScope applicationScope, ConsistencyLevel consistencyLevel, String type, - Collection<Field> fields ); - Collection<Field> fields, boolean useReadRepair) throws ConnectionException; -- ++ Collection<Field> fields, boolean useReadRepair); /** * Loads the currently persisted history of every unique value the entity has held. This will
