Merge branch 'master' into USERGRID-909 Change in logic on compaction causes issues with delete as well. Need to resolve
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/817d7ffb Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/817d7ffb Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/817d7ffb Branch: refs/heads/USERGRID-909 Commit: 817d7ffb4bcfbf43ca591a6d38d75dc0ad79743f Parents: fae5f2d 568d35d Author: Todd Nine <[email protected]> Authored: Mon Nov 2 15:55:00 2015 -0700 Committer: Todd Nine <[email protected]> Committed: Mon Nov 2 15:55:00 2015 -0700 ---------------------------------------------------------------------- README-Docs.md | 13 + README.md | 10 +- content/bootstrap/bootstrap.min.css | 9 + content/community/index.html | 22 +- content/css/bootflat-extensions.css | 356 + content/css/bootflat-square.css | 69 + content/css/bootflat.css | 1560 + content/css/font-awesome.min.css | 405 + content/css/usergrid-site.css | 1553 + .../installation/ug2-deploy-to-tomcat.txt | 42 +- .../docs/_sources/jersey2skeleton/README.txt | 9 + .../creating-and-managing-notifications.txt | 2 + .../docs/_sources/rest-endpoints/api-docs.txt | 653 +- .../docs/installation/ug2-deploy-to-tomcat.html | 37 +- content/docs/jersey2skeleton/README.html | 303 + .../creating-and-managing-notifications.html | 1 + content/docs/rest-endpoints/api-docs.html | 533 +- content/docs/searchindex.js | 2 +- content/favicon.ico | Bin 0 -> 3989 bytes content/font/FontAwesome.otf | Bin 0 -> 61896 bytes content/font/fontawesome-webfont-eot.eot | Bin 0 -> 37405 bytes content/font/fontawesome-webfont-svg.svg | 399 + content/font/fontawesome-webfont-ttf.ttf | Bin 0 -> 79076 bytes content/font/fontawesome-webfont-woff.woff | Bin 0 -> 43572 bytes content/img/alberto.jpg | Bin 0 -> 16137 bytes content/img/alex.png | Bin 0 -> 40842 bytes content/img/apache_usergrid_favicon.png | Bin 0 -> 10735 bytes content/img/apache_usergrid_logo_white.png | Bin 0 -> 26418 bytes .../img/apache_usergrid_logo_white_small.png | Bin 0 -> 11905 bytes content/img/check_flat/default.png | Bin 0 -> 25851 bytes content/img/dave.jpg | Bin 0 -> 14005 bytes content/img/ed.jpg | Bin 0 -> 20460 bytes content/img/egg-logo.png | Bin 0 -> 9938 bytes content/img/github.png | Bin 0 -> 8936 bytes content/img/grey.png | Bin 0 -> 37896 bytes content/img/intellij.png | Bin 0 -> 9199 bytes content/img/jeff.jpg | Bin 0 -> 13857 bytes content/img/michael_r.jpg | Bin 0 -> 10244 bytes content/img/mike_d.JPG | Bin 0 -> 36443 bytes content/img/nate.jpg | Bin 0 -> 4291 bytes content/img/rod.jpg | Bin 0 -> 40313 bytes content/img/scott.jpg | Bin 0 -> 8555 bytes content/img/shawn.jpg | Bin 0 -> 69304 bytes content/img/stliu.jpg | Bin 0 -> 51303 bytes content/img/strong.jpg | Bin 0 -> 7434 bytes content/img/structure101.png | Bin 0 -> 6475 bytes content/img/sungju.jpg | Bin 0 -> 11440 bytes content/img/tim.jpg | Bin 0 -> 7611 bytes content/img/todd.jpg | Bin 0 -> 18142 bytes content/img/usergrid-logo.pdf | 398 + content/img/usergrid.png | Bin 0 -> 21994 bytes content/img/usergrid_160.png | Bin 0 -> 2126 bytes content/img/usergrid_200.png | Bin 0 -> 6397 bytes content/img/usergrid_300.png | Bin 0 -> 16330 bytes content/img/usergrid_300_transparent.png | Bin 0 -> 16308 bytes content/img/usergrid_400.png | Bin 0 -> 8746 bytes content/img/usergrid_800.png | Bin 0 -> 14452 bytes content/img/usergrid_card.png | Bin 0 -> 23295 bytes content/img/usergrid_logo.png | Bin 0 -> 118086 bytes content/img/usergrid_logo_205_50.png | Bin 0 -> 7058 bytes content/img/usergrid_logo_260_50.png | Bin 0 -> 8682 bytes content/img/usergrid_logo_720.png | Bin 0 -> 27610 bytes content/img/usergrid_logo_720p.png | Bin 0 -> 27608 bytes content/img/usergrid_logo_900_200.png | Bin 0 -> 12273 bytes content/img/usergrid_logo_white.png | Bin 0 -> 16900 bytes content/img/usergrid_profile_128.png | Bin 0 -> 6689 bytes content/img/usergrid_profile_256.png | Bin 0 -> 10470 bytes content/img/usergrid_profile_256_white.png | Bin 0 -> 10724 bytes content/img/usergrid_profile_512_margins.png | Bin 0 -> 19112 bytes content/img/usergrid_profile_64_white.png | Bin 0 -> 4839 bytes content/img/usergrid_profile_background.png | Bin 0 -> 6710 bytes content/img/usergrid_screencast_bg.png | Bin 0 -> 20478 bytes content/img/usergrid_small_100.png | Bin 0 -> 6681 bytes content/img/usergrid_wiki.png | Bin 0 -> 7135 bytes content/img/yourkit.jpeg | Bin 0 -> 7763 bytes content/js/bootstrap.min.js | 8 + content/js/head.js | 708 + content/js/html5shiv.js | 8 + content/js/jquery-1.10.1.min.js | 6 + content/js/jquery.icheck.js | 397 + content/js/respond.min.js | 6 + content/js/usergrid-site.js | 50 + content/static/github-btn.html | 2 + content/v101-portal-demo/config.js | 129 + content/v101-portal-demo/css/entypo/entypo.eot | Bin 0 -> 35540 bytes content/v101-portal-demo/css/entypo/entypo.svg | 13 + content/v101-portal-demo/css/entypo/entypo.ttf | Bin 0 -> 35392 bytes content/v101-portal-demo/css/entypo/entypo.woff | Bin 0 -> 21916 bytes content/v101-portal-demo/css/main.css | 1990 + content/v101-portal-demo/css/main.min.css | 1 + content/v101-portal-demo/favicon.ico | Bin 0 -> 3989 bytes content/v101-portal-demo/helpJson.json | 47 + .../img/appswitcher/apiPlatform_lg.png | Bin 0 -> 2397 bytes .../img/appswitcher/appServices_lg.png | Bin 0 -> 2295 bytes .../img/appswitcher/console_lg.png | Bin 0 -> 1453 bytes .../img/appswitcher/home_lg.png | Bin 0 -> 1522 bytes .../img/appswitcher/logo_color.png | Bin 0 -> 3459 bytes .../v101-portal-demo/img/appswitcher/max_lg.png | Bin 0 -> 1970 bytes .../img/appswitcher/triangleMenuItem_right.png | Bin 0 -> 1158 bytes .../triangleMenuItem_right_hover.png | Bin 0 -> 1169 bytes content/v101-portal-demo/img/blue-bars.png | Bin 0 -> 3635 bytes content/v101-portal-demo/img/blue-bolt.png | Bin 0 -> 3942 bytes content/v101-portal-demo/img/blue-carat.png | Bin 0 -> 1006 bytes content/v101-portal-demo/img/green_dot.png | Bin 0 -> 3472 bytes .../img/introjs_arrow_step_next.png | Bin 0 -> 219 bytes .../img/introjs_arrow_step_next_disabled.png | Bin 0 -> 220 bytes .../img/introjs_arrow_step_prev.png | Bin 0 -> 217 bytes .../img/introjs_arrow_step_prev_disabled.png | Bin 0 -> 218 bytes content/v101-portal-demo/img/introjs_close.png | Bin 0 -> 274 bytes content/v101-portal-demo/img/logo.png | Bin 0 -> 7758 bytes content/v101-portal-demo/img/nav-device.gif | Bin 0 -> 2184 bytes content/v101-portal-demo/img/nav-sprites.png | Bin 0 -> 7953 bytes content/v101-portal-demo/img/no-data1.png | Bin 0 -> 45300 bytes content/v101-portal-demo/img/phone-small.gif | Bin 0 -> 1300 bytes .../img/push/APNS_cert_upload.png | Bin 0 -> 33956 bytes .../img/push/APNS_certification.png | Bin 0 -> 16855 bytes .../img/push/android-notification.png | Bin 0 -> 41629 bytes .../img/push/google_api_key.png | Bin 0 -> 98118 bytes .../img/push/iphone_message.png | Bin 0 -> 90307 bytes content/v101-portal-demo/img/push/step_1.png | Bin 0 -> 1953 bytes content/v101-portal-demo/img/push/step_2.png | Bin 0 -> 2117 bytes content/v101-portal-demo/img/push/step_3.png | Bin 0 -> 2162 bytes content/v101-portal-demo/img/red_dot.png | Bin 0 -> 3482 bytes .../v101-portal-demo/img/sdk-sprites-large.png | Bin 0 -> 15115 bytes content/v101-portal-demo/img/sdk-sprites.png | Bin 0 -> 4401 bytes content/v101-portal-demo/img/tablet-small.gif | Bin 0 -> 1390 bytes content/v101-portal-demo/img/user-photo.png | Bin 0 -> 3849 bytes content/v101-portal-demo/img/user_profile.png | Bin 0 -> 3775 bytes content/v101-portal-demo/img/verify.png | Bin 0 -> 22934 bytes content/v101-portal-demo/img/yellow_dot.png | Bin 0 -> 3475 bytes content/v101-portal-demo/index-debug.html | 151 + content/v101-portal-demo/index-template.html | 156 + content/v101-portal-demo/index.html | 151 + .../js/generated/usergrid-dev.js | 4886 + .../js/generated/usergrid-libs.min.js | 38 + .../js/generated/usergrid.min.js | 25 + content/v101-portal-demo/js/libs/MD5.min.js | 1 + .../js/libs/angular-1.2.5/angular-animate.js | 1323 + .../libs/angular-1.2.5/angular-animate.min.js | 23 + .../angular-1.2.5/angular-animate.min.js.map | 8 + .../js/libs/angular-1.2.5/angular-cookies.js | 202 + .../libs/angular-1.2.5/angular-cookies.min.js | 8 + .../angular-1.2.5/angular-cookies.min.js.map | 8 + .../js/libs/angular-1.2.5/angular-csp.css | 24 + .../js/libs/angular-1.2.5/angular-loader.js | 410 + .../js/libs/angular-1.2.5/angular-loader.min.js | 9 + .../angular-1.2.5/angular-loader.min.js.map | 8 + .../js/libs/angular-1.2.5/angular-mocks.js | 2116 + .../js/libs/angular-1.2.5/angular-resource.js | 565 + .../libs/angular-1.2.5/angular-resource.min.js | 13 + .../angular-1.2.5/angular-resource.min.js.map | 8 + .../js/libs/angular-1.2.5/angular-route.js | 911 + .../js/libs/angular-1.2.5/angular-route.min.js | 14 + .../libs/angular-1.2.5/angular-route.min.js.map | 8 + .../js/libs/angular-1.2.5/angular-sanitize.js | 622 + .../libs/angular-1.2.5/angular-sanitize.min.js | 14 + .../angular-1.2.5/angular-sanitize.min.js.map | 8 + .../js/libs/angular-1.2.5/angular-scenario.js | 32374 ++++++ .../js/libs/angular-1.2.5/angular-touch.js | 563 + .../js/libs/angular-1.2.5/angular-touch.min.js | 13 + .../libs/angular-1.2.5/angular-touch.min.js.map | 8 + .../js/libs/angular-1.2.5/angular.js | 20369 ++++ .../js/libs/angular-1.2.5/angular.min.js | 201 + .../js/libs/angular-1.2.5/angular.min.js.map | 8 + .../js/libs/angular-1.2.5/errors.json | 1 + .../js/libs/angular-1.2.5/version.json | 1 + .../js/libs/angular-1.2.5/version.txt | 1 + .../libs/bootstrap/css/bootstrap-responsive.css | 1345 + .../bootstrap/css/bootstrap-responsive.min.css | 1245 + .../js/libs/bootstrap/css/bootstrap.css | 6169 ++ .../js/libs/bootstrap/css/bootstrap.min.css | 5469 ++ .../js/libs/bootstrap/custom/css/bootstrap.css | 6316 ++ .../libs/bootstrap/custom/css/bootstrap.min.css | 9 + .../custom/img/glyphicons-halflings-white.png | Bin 0 -> 8777 bytes .../custom/img/glyphicons-halflings.png | Bin 0 -> 12799 bytes .../js/libs/bootstrap/custom/js/bootstrap.js | 2291 + .../libs/bootstrap/custom/js/bootstrap.min.js | 7 + .../img/glyphicons-halflings-white.png | Bin 0 -> 8777 bytes .../libs/bootstrap/img/glyphicons-halflings.png | Bin 0 -> 12799 bytes .../js/libs/bootstrap/js/bootstrap.js | 2117 + .../js/libs/bootstrap/js/bootstrap.min.js | 644 + .../v101-portal-demo/js/libs/google-viz-api.js | 49 + .../js/libs/jquery/jquery-1.9.1.min.js | 5 + .../js/libs/jquery/jquery-migrate-1.1.1.min.js | 3 + .../js/libs/jquery/jquery.sparkline.min.js | 5 + .../js/libs/jqueryui/date.min.js | 2 + .../ui-bg_diagonals-thick_90_eeeeee_40x40.png | Bin 0 -> 251 bytes .../images/ui-bg_flat_100_deedf7_40x100.png | Bin 0 -> 182 bytes .../images/ui-bg_flat_100_e4f1fb_40x100.png | Bin 0 -> 213 bytes .../images/ui-bg_flat_100_f2f5f7_40x100.png | Bin 0 -> 212 bytes .../images/ui-bg_flat_15_cd0a0a_40x100.png | Bin 0 -> 181 bytes .../images/ui-bg_flat_50_3baae3_40x100.png | Bin 0 -> 182 bytes .../images/ui-bg_flat_80_d7ebf9_40x100.png | Bin 0 -> 183 bytes .../ui-bg_highlight-hard_70_000000_1x100.png | Bin 0 -> 118 bytes .../ui-bg_highlight-soft_25_ffef8f_1x100.png | Bin 0 -> 153 bytes .../jqueryui/images/ui-icons_000000_256x240.png | Bin 0 -> 4369 bytes .../jqueryui/images/ui-icons_2694e8_256x240.png | Bin 0 -> 4369 bytes .../jqueryui/images/ui-icons_2e83ff_256x240.png | Bin 0 -> 4369 bytes .../jqueryui/images/ui-icons_3d80b3_256x240.png | Bin 0 -> 4369 bytes .../jqueryui/images/ui-icons_72a7cf_256x240.png | Bin 0 -> 4369 bytes .../jqueryui/images/ui-icons_ffffff_256x240.png | Bin 0 -> 4369 bytes .../js/libs/jqueryui/jquery-ui-1.8.18.min.js | 15 + .../js/libs/jqueryui/jquery-ui-1.8.9.custom.css | 1 + .../js/libs/jqueryui/jquery-ui-timepicker.css | 1 + .../libs/jqueryui/jquery.ui.timepicker.min.js | 1 + .../ui-bootstrap-custom-0.3.0.min.js | 1 + .../ui-bootstrap-custom-tpls-0.3.0.min.js | 1 + .../v101-portal-demo/js/libs/usergrid.sdk.js | 2568 + docs/installation/ug2-deploy-to-tomcat.md | 42 +- docs/jersey2skeleton/README.md | 9 + docs/jersey2skeleton/pom.xml | 387 + .../usergrid/testapp/AbstractResource.java | 51 + .../apache/usergrid/testapp/ApiResponse.java | 52 + .../org/apache/usergrid/testapp/Entity.java | 55 + .../usergrid/testapp/ManagementResource.java | 56 + .../usergrid/testapp/OrganizationResource.java | 64 + .../usergrid/testapp/OrganizationsResource.java | 76 + .../apache/usergrid/testapp/StatusResource.java | 34 + .../java/org/apache/usergrid/testapp/User.java | 28 + .../src/main/resources/applicationContext.xml | 28 + .../main/resources/jersey2skeleton-context.xml | 28 + .../src/main/webapp/WEB-INF/web.xml | 95 + .../usergrid/testapp/AvailablePortFinder.java | 208 + .../usergrid/testapp/OrganizationTest.java | 127 + .../org/apache/usergrid/testapp/TomcatMain.java | 57 + .../apache/usergrid/testapp/TomcatRuntime.java | 185 + .../src/test/resources/log4j.properties | 31 + .../src/test/resources/logging.properties | 26 + docs/jersey2skeleton/templates/markdown.hbs | 108 + docs/jersey2skeleton/templates/operation.hbs | 73 + docs/jersey2skeleton/templates/security.hbs | 88 + .../templates/strapdown.html.hbs | 10 + docs/rest-endpoints/api-docs.html | 750 +- docs/rest-endpoints/api-docs.md | 142 +- stack/Coverage.md | 40 + stack/awscluster/gatling-cluster-cf.json | 2 +- stack/build-tools/pom.xml | 2 +- stack/config/pom.xml | 2 +- .../main/resources/usergrid-default.properties | 6 + stack/core/pom.xml | 93 +- .../batch/service/JobSchedulerService.java | 2 +- .../usergrid/corepersistence/CoreModule.java | 6 +- .../corepersistence/CpEntityManager.java | 84 +- .../corepersistence/CpEntityManagerFactory.java | 10 +- .../corepersistence/CpManagerCache.java | 12 +- .../corepersistence/CpRelationManager.java | 61 +- .../usergrid/corepersistence/ManagerCache.java | 7 + .../asyncevents/AmazonAsyncEventService.java | 476 +- .../asyncevents/AsyncEventService.java | 6 + .../asyncevents/AsyncIndexProvider.java | 28 +- .../asyncevents/EventBuilder.java | 25 +- .../asyncevents/EventBuilderImpl.java | 13 +- .../asyncevents/InMemoryAsyncEventService.java | 134 - .../asyncevents/model/AsyncEvent.java | 22 +- .../asyncevents/model/EdgeDeleteEvent.java | 6 +- .../asyncevents/model/EdgeIndexEvent.java | 9 +- .../model/ElasticsearchIndexEvent.java | 51 + .../asyncevents/model/EntityDeleteEvent.java | 8 +- .../asyncevents/model/EntityIndexEvent.java | 6 +- .../model/InitializeApplicationIndexEvent.java | 4 +- .../index/IndexProcessorFig.java | 51 +- .../corepersistence/index/IndexServiceImpl.java | 21 +- .../index/ReIndexServiceImpl.java | 2 +- .../migration/DeDupConnectionDataMigration.java | 2 +- .../pipeline/read/FilterResult.java | 7 + .../read/search/CandidateEntityFilter.java | 17 +- .../pipeline/read/search/CandidateIdFilter.java | 15 +- .../read/traverse/AbstractReadGraphFilter.java | 136 +- .../read/traverse/EdgeCursorSerializer.java | 8 +- .../read/traverse/EntityLoadVerifyFilter.java | 24 +- .../traverse/ReadGraphCollectionFilter.java | 11 +- .../ReadGraphConnectionByTypeFilter.java | 11 +- .../traverse/ReadGraphConnectionFilter.java | 11 +- .../results/ObservableQueryExecutor.java | 7 + .../service/StatusServiceImpl.java | 3 +- .../util/ObjectJsonSerializer.java | 92 + .../usergrid/persistence/CredentialsInfo.java | 46 + .../usergrid/persistence/EntityManager.java | 7 +- .../persistence/EntityManagerFactory.java | 4 +- .../persistence/ObservableIterator.java | 83 - .../org/apache/usergrid/CoreApplication.java | 11 +- .../corepersistence/StaleIndexCleanupTest.java | 56 +- .../index/AmazonAsyncEventServiceTest.java | 13 +- .../index/AsyncIndexServiceTest.java | 5 +- .../index/InMemoryAsycIndexServiceTest.java | 65 - .../corepersistence/index/IndexServiceTest.java | 30 +- .../corepersistence/index/PublishRxTest.java | 95 - .../usergrid/corepersistence/index/RxTest.java | 108 + .../pipeline/cursor/CursorTest.java | 24 +- .../service/ConnectionServiceImplTest.java | 5 +- .../persistence/ApplicationServiceIT.java | 13 +- .../usergrid/persistence/CollectionIT.java | 10 +- .../usergrid/persistence/PermissionsIT.java | 7 +- .../query/IntersectionTransitivePagingIT.java | 1 + .../persistence/query/IteratingQueryIT.java | 5 +- stack/corepersistence/cache/pom.xml | 69 + .../persistence/cache/CacheFactory.java | 27 + .../usergrid/persistence/cache/CacheScope.java | 38 + .../usergrid/persistence/cache/ScopedCache.java | 39 + .../persistence/cache/guice/CacheModule.java | 51 + .../cache/impl/CacheFactoryImpl.java | 61 + .../persistence/cache/impl/ScopedCacheImpl.java | 57 + .../cache/impl/ScopedCacheSerialization.java | 37 + .../impl/ScopedCacheSerializationImpl.java | 310 + .../persistence/cache/CacheTestModule.java | 48 + .../persistence/cache/ScopedCacheTest.java | 173 + stack/corepersistence/collection/pom.xml | 83 +- .../cache/CachedEntityCollectionManager.java | 147 - .../EntityCollectionManagerFactoryImpl.java | 6 - stack/corepersistence/common/pom.xml | 20 +- .../core/astyanax/CassandraConfig.java | 8 +- .../core/astyanax/CassandraConfigImpl.java | 1 + .../persistence/core/astyanax/CassandraFig.java | 8 +- .../core/astyanax/MultiRowColumnIterator.java | 46 +- .../core/future/FutureObservable.java | 44 - .../persistence/core/rx/ObservableIterator.java | 2 +- .../core/rx/ExceptionBehaviorTest.java | 197 + stack/corepersistence/graph/pom.xml | 83 +- .../usergrid/persistence/graph/GraphFig.java | 3 +- .../persistence/graph/GraphManager.java | 14 +- .../usergrid/persistence/graph/MarkedEdge.java | 15 +- .../persistence/graph/guice/GraphModule.java | 9 +- .../graph/impl/GraphManagerImpl.java | 90 +- .../graph/impl/SimpleMarkedEdge.java | 62 +- .../impl/stage/NodeDeleteListenerImpl.java | 27 +- .../impl/EdgeSerializationImpl.java | 18 +- .../impl/shard/AsyncTaskExecutor.java | 34 + .../graph/serialization/impl/shard/Shard.java | 15 + .../impl/shard/ShardEntryGroup.java | 9 + .../impl/shard/ShardGroupCompaction.java | 4 - .../impl/shard/ShardGroupDeletion.java | 78 + .../impl/shard/impl/AsyncTaskExecutorImpl.java | 53 + .../shard/impl/NodeShardAllocationImpl.java | 25 +- .../shard/impl/ShardGroupColumnIterator.java | 72 +- .../shard/impl/ShardGroupCompactionImpl.java | 8 +- .../impl/shard/impl/ShardGroupDeletionImpl.java | 230 + .../impl/shard/impl/ShardsColumnIterator.java | 10 + .../persistence/graph/GraphManagerIT.java | 76 +- .../persistence/graph/GraphManagerLoadTest.java | 10 +- .../graph/GraphManagerShardConsistencyIT.java | 304 +- .../graph/GraphManagerStressTest.java | 16 +- .../impl/shard/ShardEntryGroupTest.java | 14 + .../shard/impl/ShardGroupDeletionImplTest.java | 341 + stack/corepersistence/map/pom.xml | 22 +- .../usergrid/persistence/map/MapManager.java | 25 +- .../persistence/map/impl/MapManagerImpl.java | 6 + .../persistence/map/impl/MapSerialization.java | 27 +- .../map/impl/MapSerializationImpl.java | 227 +- .../persistence/map/guice/TestMapModule.java | 18 + stack/corepersistence/model/pom.xml | 22 +- stack/corepersistence/pom.xml | 109 +- stack/corepersistence/queryindex/pom.xml | 52 +- .../persistence/index/CandidateResult.java | 11 +- .../usergrid/persistence/index/EntityIndex.java | 20 +- .../persistence/index/EntityIndexBatch.java | 8 +- .../usergrid/persistence/index/IndexFig.java | 27 +- .../persistence/index/IndexRefreshCommand.java | 50 - .../persistence/index/guice/IndexModule.java | 3 +- .../index/impl/DeIndexOperation.java | 9 + .../index/impl/EsEntityIndexBatchImpl.java | 53 +- .../index/impl/EsEntityIndexFactoryImpl.java | 10 +- .../index/impl/EsEntityIndexImpl.java | 233 +- .../index/impl/EsIndexBufferConsumerImpl.java | 286 - .../index/impl/EsIndexProducerImpl.java | 235 + .../index/impl/IndexBufferConsumer.java | 36 - .../persistence/index/impl/IndexOperation.java | 4 + .../index/impl/IndexOperationMessage.java | 30 +- .../persistence/index/impl/IndexProducer.java | 39 + .../index/impl/IndexRefreshCommandImpl.java | 202 - .../persistence/index/impl/IndexingUtils.java | 2 +- .../EsIndexMappingMigrationPlugin.java | 2 +- .../persistence/index/impl/EntityIndexTest.java | 86 +- .../persistence/index/impl/GeoPagingTest.java | 6 +- .../index/impl/IndexLoadTestsIT.java | 5 +- stack/corepersistence/queue/pom.xml | 21 +- .../persistence/queue/DefaultQueueManager.java | 75 - .../persistence/queue/LocalQueueManager.java | 108 + .../usergrid/persistence/queue/QueueFig.java | 14 +- .../persistence/queue/QueueManager.java | 15 +- .../persistence/queue/QueueMessage.java | 10 + .../persistence/queue/guice/QueueModule.java | 1 - .../queue/impl/QueueManagerFactoryImpl.java | 4 +- .../queue/impl/SNSQueueManagerImpl.java | 622 +- .../queue/impl/SQSQueueManagerImpl.java | 344 - .../queue/util/AmazonNotificationUtils.java | 3 + .../persistence/queue/QueueManagerTest.java | 47 +- stack/jacoco-pom.xml | 122 + stack/loadtests/README.md | 14 +- stack/loadtests/runAuditDeleteEntities.sh | 65 + .../runAuditGetAllAppCollectionEntities.sh | 6 +- .../loadtests/runAuditGetCollectionEntities.sh | 6 +- .../runAuditVerifyCollectionEntities.sh | 11 +- stack/loadtests/runDeleteEntities.sh | 13 +- stack/loadtests/runLoadEntitiesUnlimited.sh | 104 + stack/loadtests/runOrgAppSetup.sh | 17 +- .../datagenerators/FeederGenerator.scala | 25 +- .../usergrid/enums/ConfigProperties.scala | 19 +- .../apache/usergrid/enums/ScenarioType.scala | 3 +- .../apache/usergrid/helpers/Extractors.scala | 38 +- .../org/apache/usergrid/helpers/Headers.scala | 7 + .../usergrid/scenarios/AuditScenarios.scala | 226 +- .../scenarios/EntityCollectionScenarios.scala | 102 +- .../org/apache/usergrid/settings/Settings.scala | 174 +- .../usergrid/simulations/AuditSimulation.scala | 15 +- .../simulations/ConfigurableSimulation.scala | 2 +- stack/loadtests/testConfig.sh | 7 +- stack/pom.xml | 280 +- stack/rest/pom.xml | 156 +- .../usergrid/rest/AbstractContextResource.java | 45 +- .../usergrid/rest/ApplicationsResource.java | 168 - .../apache/usergrid/rest/DatabaseResource.java | 98 - .../org/apache/usergrid/rest/IndexResource.java | 328 - .../apache/usergrid/rest/MigrateResource.java | 268 - .../org/apache/usergrid/rest/RootResource.java | 92 +- .../apache/usergrid/rest/SwaggerServlet.java | 5 +- .../apache/usergrid/rest/SystemResource.java | 103 - .../usergrid/rest/TextToJsonSwapWriter.java | 182 +- .../rest/applications/ApplicationResource.java | 80 +- .../rest/applications/ServiceResource.java | 111 +- .../applications/assets/AssetsResource.java | 13 +- .../applications/events/EventsResource.java | 36 +- .../notifiers/NotifierResource.java | 29 +- .../notifiers/NotifiersResource.java | 29 +- .../rest/applications/queues/QueueResource.java | 86 +- .../queues/QueueSubscriberResource.java | 63 +- .../queues/QueueSubscriptionResource.java | 63 +- .../queues/QueueTransactionsResource.java | 40 +- .../rest/applications/users/UserResource.java | 172 +- .../rest/applications/users/UsersResource.java | 51 +- .../exceptions/AbstractExceptionMapper.java | 2 +- .../exceptions/NotFoundExceptionMapper.java | 3 +- .../rest/exceptions/SecurityException.java | 31 +- .../exceptions/SecurityExceptionMapper.java | 3 +- .../exceptions/UnsupportedMediaTypeMapper.java | 43 + .../rest/filters/ContentTypeFilter.java | 74 +- .../rest/filters/JSONPCallbackFilter.java | 26 +- .../usergrid/rest/filters/MeteringFilter.java | 632 +- .../usergrid/rest/filters/TracingFilter.java | 37 +- .../rest/management/ManagementResource.java | 53 +- .../management/metrics/MetricsResource.java | 9 +- .../organizations/OrganizationResource.java | 64 +- .../organizations/OrganizationsResource.java | 97 +- .../applications/ApplicationResource.java | 56 +- .../applications/ApplicationsResource.java | 22 +- .../imports/FileErrorsResource.java | 41 +- .../imports/FileIncludesResource.java | 15 +- .../applications/imports/ImportsResource.java | 57 +- .../organizations/users/UsersResource.java | 88 +- .../rest/management/users/UserResource.java | 95 +- .../rest/management/users/UsersResource.java | 11 +- .../organizations/OrganizationsResource.java | 76 +- .../organizations/OrganizationResource.java | 28 +- .../rest/security/CrossOriginRequestFilter.java | 18 +- .../security/SecuredResourceFilterFactory.java | 182 +- .../ShiroAuthenticationExceptionMapper.java | 3 +- .../shiro/filters/BasicAuthSecurityFilter.java | 35 +- .../ClientCredentialsSecurityFilter.java | 25 +- .../OAuth2AccessTokenSecurityFilter.java | 168 +- .../security/shiro/filters/SecurityFilter.java | 33 +- .../session/HttpRequestSessionManager.java | 9 +- .../rest/system/ApplicationsResource.java | 178 + .../rest/system/ConnectionResource.java | 216 + .../usergrid/rest/system/DatabaseResource.java | 103 + .../usergrid/rest/system/IndexResource.java | 353 + .../usergrid/rest/system/MigrateResource.java | 277 + .../rest/system/QueueSystemResource.java | 77 + .../usergrid/rest/system/SystemResource.java | 121 + .../apache/usergrid/rest/utils/CORSUtils.java | 42 +- .../main/resources/usergrid-rest-context.xml | 2 - stack/rest/src/main/webapp/WEB-INF/web.xml | 63 +- .../java/org/apache/usergrid/rest/BasicIT.java | 22 +- .../usergrid/rest/CollectionMetadataIT.java | 7 +- .../apache/usergrid/rest/IndexResourceIT.java | 43 +- .../apache/usergrid/rest/NotificationsIT.java | 6 +- .../apache/usergrid/rest/PartialUpdateTest.java | 16 +- .../apache/usergrid/rest/SystemResourceIT.java | 27 +- .../org/apache/usergrid/rest/TomcatRuntime.java | 12 +- .../rest/applications/ApplicationCreateIT.java | 14 +- .../rest/applications/ApplicationDeleteIT.java | 206 +- .../applications/ApplicationResourceIT.java | 267 +- .../applications/assets/AssetResourceIT.java | 21 +- .../applications/assets/AwsAssetResourceIT.java | 53 +- .../applications/assets/aws/NoAWSCredsRule.java | 15 +- .../collection/CollectionsResourceIT.java | 26 +- .../collection/DuplicateNameIT.java | 5 +- .../activities/ActivityResourceIT.java | 5 +- .../collection/devices/DevicesResourceIT.java | 4 +- .../collection/groups/GroupResourceIT.java | 34 +- .../collection/paging/PagingResourceIT.java | 65 +- .../users/ConnectionResourceTest.java | 50 +- .../collection/users/OwnershipResourceIT.java | 15 +- .../collection/users/PermissionsResourceIT.java | 93 +- .../collection/users/UserResourceIT.java | 243 +- .../queries/BadGrammarQueryTest.java | 15 +- .../applications/queries/BasicGeoTests.java | 45 +- .../rest/filters/ContentTypeResourceIT.java | 70 +- .../usergrid/rest/management/AccessTokenIT.java | 42 +- .../usergrid/rest/management/AdminUsersIT.java | 48 +- .../rest/management/ExportResourceIT.java | 137 +- .../rest/management/ImportResourceIT.java | 50 +- .../rest/management/ManagementResourceIT.java | 149 +- .../rest/management/OrganizationsIT.java | 60 +- .../rest/management/RegistrationIT.java | 32 +- .../organizations/AdminEmailEncodingIT.java | 5 +- .../rest/test/resource/AbstractRestIT.java | 79 +- .../rest/test/resource/ClientSetup.java | 7 +- .../usergrid/rest/test/resource/RestClient.java | 44 +- .../test/resource/TestPropertiesResource.java | 14 +- .../endpoints/ApplicationsResource.java | 8 +- .../resource/endpoints/BoostrapResource.java | 19 +- .../resource/endpoints/CollectionEndpoint.java | 109 +- .../test/resource/endpoints/EntityEndpoint.java | 40 +- .../test/resource/endpoints/NamedResource.java | 241 +- .../test/resource/endpoints/RootResource.java | 9 +- .../test/resource/endpoints/SetupResource.java | 20 +- .../test/resource/endpoints/SubResource.java | 37 + .../test/resource/endpoints/SystemResource.java | 51 +- .../test/resource/endpoints/TokenResource.java | 15 +- .../test/resource/endpoints/UrlResource.java | 11 +- .../endpoints/mgmt/ApplicationResource.java | 27 +- .../endpoints/mgmt/ApplicationsResource.java | 4 +- .../endpoints/mgmt/AuthorizeResource.java | 7 +- .../endpoints/mgmt/ConfirmResource.java | 12 +- .../endpoints/mgmt/CredentialsResource.java | 7 +- .../resource/endpoints/mgmt/FeedResource.java | 5 +- .../resource/endpoints/mgmt/OrgResource.java | 68 +- .../endpoints/mgmt/OrganizationResource.java | 15 +- .../endpoints/mgmt/PasswordResource.java | 8 - .../endpoints/mgmt/ReactivateResource.java | 10 +- .../resource/endpoints/mgmt/ResetResource.java | 10 +- .../resource/endpoints/mgmt/TokenResource.java | 7 +- .../resource/endpoints/mgmt/UserResource.java | 20 +- .../resource/endpoints/mgmt/UsersResource.java | 6 - stack/rest/src/test/resources/log4j.properties | 39 +- .../rest/src/test/resources/logging.properties | 13 +- stack/rest_integration_tests/README.md | 19 - stack/rest_integration_tests/config/default.js | 37 - stack/rest_integration_tests/config/index.js | 25 - stack/rest_integration_tests/index.js | 20 - stack/rest_integration_tests/lib/connections.js | 137 - stack/rest_integration_tests/lib/entities.js | 125 - .../rest_integration_tests/lib/notifications.js | 68 - stack/rest_integration_tests/lib/random.js | 74 - stack/rest_integration_tests/lib/response.js | 30 - stack/rest_integration_tests/lib/token.js | 49 - stack/rest_integration_tests/lib/urls.js | 37 - stack/rest_integration_tests/lib/users.js | 82 - stack/rest_integration_tests/package.json | 18 - .../test/authentication/management.js | 38 - .../test/authentication/org.js | 37 - .../test/authentication/resetPassword.js | 78 - .../test/authentication/user.js | 40 - .../test/connections/create.js | 49 - .../test/connections/delete.js | 77 - .../test/connections/get.js | 82 - .../test/entities/create.js | 38 - .../test/entities/deleteAll.js | 37 - .../rest_integration_tests/test/entities/get.js | 51 - .../test/entities/update.js | 43 - stack/rest_integration_tests/test/main.js | 72 - stack/rest_integration_tests/test/mocha.opts | 3 - .../test/notifications/create.js | 36 - .../test/queries/comparison.js | 58 - .../test/queries/contains.js | 116 - .../test/queries/equals.js | 54 - .../test/queries/location.js | 42 - .../test/queries/order.js | 82 - stack/rest_integration_tests/test/setup.js | 103 - stack/rest_integration_tests/test/teardown.js | 65 - .../rest_integration_tests/test/users/create.js | 45 - stack/scripts/create_test_data.py | 222 + stack/scripts/migrate_entity_data.py | 18 +- stack/scripts/multitenant_migrate.py | 683 + stack/services/pom.xml | 83 +- .../management/AppInfoMigrationPlugin.java | 18 +- .../usergrid/management/ApplicationInfo.java | 6 +- .../usergrid/management/ManagementService.java | 14 +- .../apache/usergrid/management/UserInfo.java | 25 +- .../cassandra/ManagementServiceImpl.java | 60 +- .../security/oauth/ClientCredentialsInfo.java | 6 +- .../security/providers/AbstractProvider.java | 16 +- .../security/providers/FacebookProvider.java | 7 +- .../security/providers/FoursquareProvider.java | 38 +- .../providers/PingIdentityProvider.java | 38 +- .../apache/usergrid/security/shiro/Realm.java | 444 +- .../usergrid/security/shiro/ShiroCache.java | 171 + .../security/shiro/ShiroCacheManager.java | 114 + .../shiro/UsergridAuthenticationInfo.java | 283 + .../shiro/UsergridAuthorizationInfo.java | 103 + .../AbstractAccessTokenCredentials.java | 9 +- .../credentials/AbstractClientCredentials.java | 5 + .../shiro/credentials/AdminUserAccessToken.java | 2 + .../credentials/ApplicationUserAccessToken.java | 3 + .../OrganizationClientCredentials.java | 5 + .../shiro/principals/AdminUserPrincipal.java | 143 +- .../principals/ApplicationGuestPrincipal.java | 65 +- .../shiro/principals/ApplicationPrincipal.java | 39 +- .../principals/ApplicationUserPrincipal.java | 133 + .../shiro/principals/OrganizationPrincipal.java | 61 +- .../shiro/principals/PrincipalIdentifier.java | 88 + .../shiro/principals/UserPrincipal.java | 6 +- .../security/shiro/utils/SubjectUtils.java | 13 +- .../services/AbstractConnectionsService.java | 5 +- .../usergrid/services/AbstractService.java | 4 + .../assets/data/AwsSdkS3BinaryStore.java | 37 +- .../usergrid/services/groups/GroupsService.java | 10 + .../services/guice/ServiceModuleImpl.java | 51 +- .../services/notifications/QueueListener.java | 2 +- .../services/notifications/gcm/GCMAdapter.java | 7 +- .../services/queues/ImportQueueManager.java | 19 +- .../usergrid/services/queues/QueueListener.java | 3 +- .../usergrid/services/roles/RolesService.java | 20 +- .../usergrid/services/users/UsersService.java | 7 + .../services/users/roles/RolesService.java | 10 + .../resources/usergrid-services-context.xml | 11 +- .../org/apache/usergrid/ServiceApplication.java | 2 + .../usergrid/management/OrganizationIT.java | 18 +- .../org/apache/usergrid/management/RoleIT.java | 7 +- .../apns/NotificationsServiceIT.java | 4 +- .../gcm/NotificationsServiceIT.java | 2 - .../resources/usergrid-custom-test.properties | 3 + stack/test-utils/pom.xml | 89 +- stack/tools/pom.xml | 2 +- .../org/apache/usergrid/tools/ExportAdmins.java | 17 +- stack/websocket/pom.xml | 2 +- website/README.md | 3 +- website/Rules | 52 + website/build.sh | 1 + website/content/bootstrap/bootstrap.min.css | 9 + website/content/community/index.html | 22 +- website/content/css/bootflat-extensions.css | 356 + website/content/css/bootflat-square.css | 69 + website/content/css/bootflat.css | 1560 + website/content/css/font-awesome.min.css | 405 + website/content/css/usergrid-site.css | 1553 + website/content/favicon.ico | Bin 0 -> 3989 bytes website/content/font/FontAwesome.otf | Bin 0 -> 61896 bytes .../content/font/fontawesome-webfont-eot.eot | Bin 0 -> 37405 bytes .../content/font/fontawesome-webfont-svg.svg | 399 + .../content/font/fontawesome-webfont-ttf.ttf | Bin 0 -> 79076 bytes .../content/font/fontawesome-webfont-woff.woff | Bin 0 -> 43572 bytes website/content/img/alberto.jpg | Bin 0 -> 16137 bytes website/content/img/alex.png | Bin 0 -> 40842 bytes website/content/img/apache_usergrid_favicon.png | Bin 0 -> 10735 bytes .../content/img/apache_usergrid_logo_white.png | Bin 0 -> 26418 bytes .../img/apache_usergrid_logo_white_small.png | Bin 0 -> 11905 bytes website/content/img/check_flat/default.png | Bin 0 -> 25851 bytes website/content/img/dave.jpg | Bin 0 -> 14005 bytes website/content/img/ed.jpg | Bin 0 -> 20460 bytes website/content/img/egg-logo.png | Bin 0 -> 9938 bytes website/content/img/github.png | Bin 0 -> 8936 bytes website/content/img/grey.png | Bin 0 -> 37896 bytes website/content/img/intellij.png | Bin 0 -> 9199 bytes website/content/img/jeff.jpg | Bin 0 -> 13857 bytes website/content/img/michael_r.jpg | Bin 0 -> 10244 bytes website/content/img/miked.jpg | Bin 0 -> 36443 bytes website/content/img/nate.jpg | Bin 0 -> 4291 bytes website/content/img/rod.jpg | Bin 0 -> 40313 bytes website/content/img/scott.jpg | Bin 0 -> 8555 bytes website/content/img/shawn.jpg | Bin 0 -> 69304 bytes website/content/img/stliu.jpg | Bin 0 -> 51303 bytes website/content/img/strong.jpg | Bin 0 -> 7434 bytes website/content/img/structure101.png | Bin 0 -> 6475 bytes website/content/img/sungju.jpg | Bin 0 -> 11440 bytes website/content/img/tim.jpg | Bin 0 -> 7611 bytes website/content/img/todd.jpg | Bin 0 -> 18142 bytes website/content/img/usergrid-logo.pdf | 398 + website/content/img/usergrid.png | Bin 0 -> 21994 bytes website/content/img/usergrid_160.png | Bin 0 -> 2126 bytes website/content/img/usergrid_200.png | Bin 0 -> 6397 bytes website/content/img/usergrid_300.png | Bin 0 -> 16330 bytes .../content/img/usergrid_300_transparent.png | Bin 0 -> 16308 bytes website/content/img/usergrid_400.png | Bin 0 -> 8746 bytes website/content/img/usergrid_800.png | Bin 0 -> 14452 bytes website/content/img/usergrid_card.png | Bin 0 -> 23295 bytes website/content/img/usergrid_logo.png | Bin 0 -> 118086 bytes website/content/img/usergrid_logo_205_50.png | Bin 0 -> 7058 bytes website/content/img/usergrid_logo_260_50.png | Bin 0 -> 8682 bytes website/content/img/usergrid_logo_720.png | Bin 0 -> 27610 bytes website/content/img/usergrid_logo_720p.png | Bin 0 -> 27608 bytes website/content/img/usergrid_logo_900_200.png | Bin 0 -> 12273 bytes website/content/img/usergrid_logo_white.png | Bin 0 -> 16900 bytes website/content/img/usergrid_profile_128.png | Bin 0 -> 6689 bytes website/content/img/usergrid_profile_256.png | Bin 0 -> 10470 bytes .../content/img/usergrid_profile_256_white.png | Bin 0 -> 10724 bytes .../img/usergrid_profile_512_margins.png | Bin 0 -> 19112 bytes .../content/img/usergrid_profile_64_white.png | Bin 0 -> 4839 bytes .../content/img/usergrid_profile_background.png | Bin 0 -> 6710 bytes website/content/img/usergrid_screencast_bg.png | Bin 0 -> 20478 bytes website/content/img/usergrid_small_100.png | Bin 0 -> 6681 bytes website/content/img/usergrid_wiki.png | Bin 0 -> 7135 bytes website/content/img/yourkit.jpeg | Bin 0 -> 7763 bytes website/content/js/bootstrap.min.js | 8 + website/content/js/head.js | 708 + website/content/js/html5shiv.js | 8 + website/content/js/jquery-1.10.1.min.js | 6 + website/content/js/jquery.icheck.js | 397 + website/content/js/respond.min.js | 6 + website/content/js/usergrid-site.js | 50 + website/content/static/github-btn.html | 2 + website/crash.log | 143 + website/layouts/community.html | 1 + website/layouts/docs.html | 1 + website/lib/default.rb | 43 + website/lib/helpers_.rb | 0 website/lib/pandoc.template | 4 + website/nanoc.yaml | 77 + website/run.sh | 1 + website/tmp/checksums | 4 +- website/tmp/compiled_content | 87099 +---------------- website/tmp/dependencies | Bin 2372 -> 2397 bytes website/tmp/rule_memory | Bin 0 -> 5163 bytes website/utilities/map-markers.rb | 62 + website/utilities/markers.txt | 440 + website/utilities/snapshot-apigee.rb | 71 + website/utilities/usergrid.csv | 290 + 716 files changed, 126888 insertions(+), 96792 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/usergrid/blob/817d7ffb/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/CassandraConfig.java ---------------------------------------------------------------------- diff --cc stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/CassandraConfig.java index bbefe0f,dba3646..0b3a38d --- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/CassandraConfig.java +++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/CassandraConfig.java @@@ -52,8 -52,7 +52,8 @@@ public interface CassandraConfig * Return the number of shards that has been set in the property file * @return */ - public int[] getShardSettings(); + int[] getShardSettings(); + } http://git-wip-us.apache.org/repos/asf/usergrid/blob/817d7ffb/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/CassandraFig.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/usergrid/blob/817d7ffb/stack/corepersistence/graph/pom.xml ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/usergrid/blob/817d7ffb/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/GraphFig.java ---------------------------------------------------------------------- diff --cc stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/GraphFig.java index 8356559,ecb5b19..82dfe51 --- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/GraphFig.java +++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/GraphFig.java @@@ -90,7 -113,27 +90,8 @@@ public interface GraphFig extends Guicy @Key( SHARD_SIZE ) long getShardSize(); - @Default( "10" ) + - @Default("30000") - @Key(SHARD_CACHE_TIMEOUT) - long getShardCacheTimeout(); - - @Default("60000") - @Key(SHARD_MIN_DELTA) - long getShardMinDelta(); - - - @Default("250000") - @Key(SHARD_CACHE_SIZE) - long getShardCacheSize(); - - - @Default("2") - @Key(SHARD_CACHE_REFRESH_WORKERS) - int getShardCacheRefreshWorkerCount(); - - + @Default( "20" ) @Key( SHARD_AUDIT_WORKERS ) int getShardAuditWorkerCount(); http://git-wip-us.apache.org/repos/asf/usergrid/blob/817d7ffb/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/GraphManager.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/usergrid/blob/817d7ffb/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/guice/GraphModule.java ---------------------------------------------------------------------- diff --cc stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/guice/GraphModule.java index 9b26fe4,d2476eb..752f417 --- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/guice/GraphModule.java +++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/guice/GraphModule.java @@@ -59,14 -59,21 +59,15 @@@ import org.apache.usergrid.persistence. import org.apache.usergrid.persistence.graph.serialization.impl.shard.EdgeShardSerialization; import org.apache.usergrid.persistence.graph.serialization.impl.shard.EdgeShardStrategy; import org.apache.usergrid.persistence.graph.serialization.impl.shard.NodeShardAllocation; -import org.apache.usergrid.persistence.graph.serialization.impl.shard.NodeShardApproximation; -import org.apache.usergrid.persistence.graph.serialization.impl.shard.NodeShardCache; +import org.apache.usergrid.persistence.graph.serialization.impl.shard.NodeShardGroupSearch; import org.apache.usergrid.persistence.graph.serialization.impl.shard.ShardGroupCompaction; + import org.apache.usergrid.persistence.graph.serialization.impl.shard.ShardGroupDeletion; import org.apache.usergrid.persistence.graph.serialization.impl.shard.ShardedEdgeSerialization; -import org.apache.usergrid.persistence.graph.serialization.impl.shard.count.NodeShardApproximationImpl; -import org.apache.usergrid.persistence.graph.serialization.impl.shard.count.NodeShardCounterSerialization; -import org.apache.usergrid.persistence.graph.serialization.impl.shard.count.NodeShardCounterSerializationImpl; -import org.apache.usergrid.persistence.graph.serialization.impl.shard.impl.AsyncTaskExecutorImpl; +import org.apache.usergrid.persistence.graph.serialization.impl.shard.ShardedEdgeSerializationImpl; import org.apache.usergrid.persistence.graph.serialization.impl.shard.impl.EdgeShardSerializationImpl; import org.apache.usergrid.persistence.graph.serialization.impl.shard.impl.NodeShardAllocationImpl; -import org.apache.usergrid.persistence.graph.serialization.impl.shard.impl.NodeShardCacheImpl; +import org.apache.usergrid.persistence.graph.serialization.impl.shard.impl.NodeShardGroupSearchImpl; import org.apache.usergrid.persistence.graph.serialization.impl.shard.impl.ShardGroupCompactionImpl; -import org.apache.usergrid.persistence.graph.serialization.impl.shard.impl.ShardGroupDeletionImpl; -import org.apache.usergrid.persistence.graph.serialization.impl.shard.impl.ShardedEdgeSerializationImpl; import org.apache.usergrid.persistence.graph.serialization.impl.shard.impl.SizebasedEdgeColumnFamilies; import org.apache.usergrid.persistence.graph.serialization.impl.shard.impl.SizebasedEdgeShardStrategy; @@@ -107,10 -114,17 +108,16 @@@ public abstract class GraphModule exten */ bind(NodeShardAllocation.class).to( NodeShardAllocationImpl.class ); - bind( NodeShardApproximation.class ).to( NodeShardApproximationImpl.class ); - bind( NodeShardCache.class ).to( NodeShardCacheImpl.class ); - bind( NodeShardCounterSerialization.class ).to( NodeShardCounterSerializationImpl.class ); + bind( NodeShardGroupSearch.class ).to( NodeShardGroupSearchImpl.class ); + /** + * Binding for task tracker + */ + bind( AsyncTaskExecutor.class ).to( AsyncTaskExecutorImpl.class ); + bind( ShardGroupDeletion.class ).to( ShardGroupDeletionImpl.class ); + + /** * Bind our strategies based on their internal annotations. */ http://git-wip-us.apache.org/repos/asf/usergrid/blob/817d7ffb/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/EdgeSerializationImpl.java ---------------------------------------------------------------------- diff --cc stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/EdgeSerializationImpl.java index 6e947b8,9e25946..1deeea6 --- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/EdgeSerializationImpl.java +++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/EdgeSerializationImpl.java @@@ -70,7 -68,7 +71,8 @@@ public class EdgeSerializationImpl impl protected final EdgeColumnFamilies edgeColumnFamilies; protected final ShardedEdgeSerialization shardedEdgeSerialization; protected final TimeService timeService; + protected final RxTaskScheduler rxTaskScheduler; + protected final ShardGroupDeletion shardGroupDeletion; @Inject @@@ -78,7 -76,8 +80,9 @@@ final GraphFig graphFig, final EdgeShardStrategy edgeShardStrategy, final EdgeColumnFamilies edgeColumnFamilies, final ShardedEdgeSerialization shardedEdgeSerialization, - final TimeService timeService, final RxTaskScheduler rxTaskScheduler ) { - final TimeService timeService, final ShardGroupDeletion shardGroupDeletion ) { ++ final TimeService timeService, final RxTaskScheduler rxTaskScheduler, ++ final ShardGroupDeletion shardGroupDeletion ) { + checkNotNull( keyspace, "keyspace required" ); @@@ -87,7 -86,7 +91,8 @@@ checkNotNull( edgeColumnFamilies, "edgeColumnFamilies required" ); checkNotNull( shardedEdgeSerialization, "shardedEdgeSerialization required" ); checkNotNull( timeService, "timeService required" ); + checkNotNull( rxTaskScheduler, "rxTaskScheduler is required" ); + checkNotNull( shardGroupDeletion, "shardGroupDeletion require"); this.keyspace = keyspace; @@@ -97,7 -96,7 +102,8 @@@ this.edgeColumnFamilies = edgeColumnFamilies; this.shardedEdgeSerialization = shardedEdgeSerialization; this.timeService = timeService; + this.rxTaskScheduler = rxTaskScheduler; + this.shardGroupDeletion = shardGroupDeletion; } @@@ -357,10 -322,9 +363,10 @@@ final DirectedEdgeMeta directedEdgeMeta = DirectedEdgeMeta.fromSourceNode( sourceId, type ); - final Iterator<ShardEntryGroup> readShards = edgeShardStrategy.getReadShards( scope, maxTimestamp, directedEdgeMeta ); + final Iterator<ShardEntryGroup> readShards = + edgeShardStrategy.getReadShards( scope, maxTimestamp, directedEdgeMeta ); - return new ShardGroupColumnIterator<MarkedEdge>( readShards ) { + return new ShardGroupColumnIterator( scope, directedEdgeMeta, shardGroupDeletion, readShards ) { @Override protected Iterator<MarkedEdge> getIterator( final Collection<Shard> readShards ) { return shardedEdgeSerialization.getEdgesFromSource( edgeColumnFamilies, scope, edgeType, readShards ); @@@ -381,15 -345,13 +387,15 @@@ final String targetType = edgeType.getIdType(); final long maxTimestamp = edgeType.getMaxTimestamp(); - final DirectedEdgeMeta directedEdgeMeta = DirectedEdgeMeta.fromSourceNodeTargetType( sourceId, type, targetType ); + final DirectedEdgeMeta directedEdgeMeta = + DirectedEdgeMeta.fromSourceNodeTargetType( sourceId, type, targetType ); - final Iterator<ShardEntryGroup> readShards = edgeShardStrategy.getReadShards( scope, maxTimestamp, directedEdgeMeta ); + final Iterator<ShardEntryGroup> readShards = + edgeShardStrategy.getReadShards( scope, maxTimestamp, directedEdgeMeta ); - return new ShardGroupColumnIterator<MarkedEdge>( readShards ) { + return new ShardGroupColumnIterator( scope, directedEdgeMeta, shardGroupDeletion, readShards ) { @Override protected Iterator<MarkedEdge> getIterator( final Collection<Shard> readShards ) { return shardedEdgeSerialization @@@ -413,10 -375,9 +419,10 @@@ final DirectedEdgeMeta directedEdgeMeta = DirectedEdgeMeta.fromTargetNode( targetId, type ); - final Iterator<ShardEntryGroup> readShards = edgeShardStrategy.getReadShards( scope, maxTimestamp, directedEdgeMeta ); + final Iterator<ShardEntryGroup> readShards = + edgeShardStrategy.getReadShards( scope, maxTimestamp, directedEdgeMeta ); - return new ShardGroupColumnIterator<MarkedEdge>( readShards ) { + return new ShardGroupColumnIterator( scope, directedEdgeMeta, shardGroupDeletion, readShards ) { @Override protected Iterator<MarkedEdge> getIterator( final Collection<Shard> readShards ) { return shardedEdgeSerialization.getEdgesToTarget( edgeColumnFamilies, scope, edgeType, readShards ); @@@ -438,15 -399,13 +444,15 @@@ final long maxTimestamp = edgeType.getMaxTimestamp(); - final DirectedEdgeMeta directedEdgeMeta = DirectedEdgeMeta.fromTargetNodeSourceType( targetId, type, sourceType ); + final DirectedEdgeMeta directedEdgeMeta = + DirectedEdgeMeta.fromTargetNodeSourceType( targetId, type, sourceType ); - final Iterator<ShardEntryGroup> readShards = edgeShardStrategy.getReadShards( scope, maxTimestamp, directedEdgeMeta ); + final Iterator<ShardEntryGroup> readShards = + edgeShardStrategy.getReadShards( scope, maxTimestamp, directedEdgeMeta ); - return new ShardGroupColumnIterator<MarkedEdge>( readShards ) { + return new ShardGroupColumnIterator( scope, directedEdgeMeta, shardGroupDeletion, readShards ) { @Override protected Iterator<MarkedEdge> getIterator( final Collection<Shard> readShards ) { return shardedEdgeSerialization http://git-wip-us.apache.org/repos/asf/usergrid/blob/817d7ffb/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/ShardEntryGroup.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/usergrid/blob/817d7ffb/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/ShardGroupCompaction.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/usergrid/blob/817d7ffb/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/NodeShardAllocationImpl.java ---------------------------------------------------------------------- diff --cc stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/NodeShardAllocationImpl.java index 79a15f4,b0875af..8f9e27f --- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/NodeShardAllocationImpl.java +++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/NodeShardAllocationImpl.java @@@ -59,11 -58,7 +59,9 @@@ import com.netflix.astyanax.util.TimeUU public class NodeShardAllocationImpl implements NodeShardAllocation { - private static final Logger LOG = LoggerFactory.getLogger( NodeShardAllocationImpl.class ); + private static final Logger logger = LoggerFactory.getLogger( NodeShardAllocationImpl.class ); + - private static final Shard MIN_SHARD = new Shard( 0, 0, true ); - + private static final NoOpCompaction NO_OP_COMPACTION = new NoOpCompaction(); private final EdgeShardSerialization edgeShardSerialization; private final EdgeColumnFamilies edgeColumnFamilies; @@@ -95,28 -93,37 +93,33 @@@ Preconditions.checkNotNull( maxShardId, "maxShardId cannot be null" ); GraphValidation.validateDirectedEdgeMeta( directedEdgeMeta ); - Iterator<Shard> existingShards; + Iterator<Shard> existingShards = null; //its a new node, it doesn't need to check cassandra, it won't exist - if ( isNewNode( directedEdgeMeta ) ) { - existingShards = Collections.singleton( Shard.MIN_SHARD ).iterator(); + if ( !isNewNode( directedEdgeMeta ) ) { + existingShards = edgeShardSerialization.getShardMetaDataLocal( scope, maxShardId, directedEdgeMeta ); } - if ( existingShards == null || !existingShards.hasNext() ) { - else { - existingShards = edgeShardSerialization.getShardMetaData( scope, maxShardId, directedEdgeMeta ); - + /** + * We didn't get anything out of cassandra, so we need to create the minumum shard + */ + if ( existingShards == null || !existingShards.hasNext() ) { - final MutationBatch batch = edgeShardSerialization.writeShardMeta( scope, MIN_SHARD, directedEdgeMeta ); - try { - batch.execute(); - } - catch ( ConnectionException e ) { - throw new RuntimeException( "Unable to connect to casandra", e ); - } + final MutationBatch batch = edgeShardSerialization.writeShardMeta( scope, Shard.MIN_SHARD, directedEdgeMeta ); + try { + batch.execute(); + } + catch ( ConnectionException e ) { + throw new RuntimeException( "Unable to connect to casandra", e ); + } - existingShards = Collections.singleton( MIN_SHARD ).iterator(); + existingShards = Collections.singleton( Shard.MIN_SHARD ).iterator(); + } } + - return new ShardEntryGroupIterator( existingShards, graphFig.getShardMinDelta(), shardGroupCompaction, scope, - directedEdgeMeta ); + return new ShardEntryGroupIterator( existingShards, shardGroupCompaction, scope, directedEdgeMeta ); } http://git-wip-us.apache.org/repos/asf/usergrid/blob/817d7ffb/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardGroupCompactionImpl.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/usergrid/blob/817d7ffb/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerShardConsistencyIT.java ---------------------------------------------------------------------- diff --cc stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerShardConsistencyIT.java index 6c2f690,6aad289..f16cc78 --- a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerShardConsistencyIT.java +++ b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerShardConsistencyIT.java @@@ -86,9 -88,7 +86,10 @@@ public class GraphManagerShardConsisten private static final Meter writeMeter = registry.meter( "writeThroughput" ); + private static final Slf4jReporter reporter = + Slf4jReporter.forRegistry( registry ).outputTo( log ).convertRatesTo( TimeUnit.SECONDS ) + .convertDurationsTo( TimeUnit.MILLISECONDS ).build(); + private Slf4jReporter reporter; protected ApplicationScope scope; @@@ -245,34 -275,24 +264,33 @@@ final List<Throwable> failures = new ArrayList<>(); - //add the future - Futures.addCallback( future, new FutureCallback<Long>() { + for ( Injector injector : injectors ) { + final GraphManagerFactory readGmf = injector.getInstance( GraphManagerFactory.class ); - @Override - public void onSuccess( @Nullable final Long result ) { - log.info( "Successfully ran the read, re-running" ); - executor.submit( new ReadWorker( gmf, generator, writeCount, readMeter ) ); - } + for ( int i = 0; i < numWorkersPerInjector; i++ ) { + final ListenableFuture<Long> future = + executor.submit( new ReadWorker( readGmf, generator, writeCount, readMeter ) ); - @Override - public void onFailure( final Throwable t ) { - failures.add( t ); - log.error( "Failed test!", t ); - } - } ); + //add the future + Futures.addCallback( future, new FutureCallback<Long>() { + @Override + public void onSuccess( @Nullable final Long result ) { + log.info( "Successfully ran the read, re-running" ); + executor.submit( new ReadWorker( gmf, generator, writeCount, readMeter ) ); + } + + + @Override + public void onFailure( final Throwable t ) { + failures.add( t ); + log.error( "Failed test!", t ); + } + } ); + } + } - int compactedCount; http://git-wip-us.apache.org/repos/asf/usergrid/blob/817d7ffb/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/ShardEntryGroupTest.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/usergrid/blob/817d7ffb/stack/corepersistence/map/src/main/java/org/apache/usergrid/persistence/map/impl/MapSerializationImpl.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/usergrid/blob/817d7ffb/stack/corepersistence/pom.xml ---------------------------------------------------------------------- diff --cc stack/corepersistence/pom.xml index 9fd67d0,dab863b..3348157 --- a/stack/corepersistence/pom.xml +++ b/stack/corepersistence/pom.xml @@@ -86,7 -105,7 +105,8 @@@ limitations under the License <module>common</module> <module>map</module> <module>queue</module> + <module>cache</module> + <module>locks</module> </modules> <build> http://git-wip-us.apache.org/repos/asf/usergrid/blob/817d7ffb/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/usergrid/blob/817d7ffb/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java ---------------------------------------------------------------------- diff --cc stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java index 5c30ea8,f03a98a..a70c63c --- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java +++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java @@@ -31,9 -31,10 +31,10 @@@ import org.apache.usergrid.persistence. import org.apache.usergrid.persistence.core.migration.data.VersionedData; import org.apache.usergrid.persistence.core.scope.ApplicationScope; import org.apache.usergrid.persistence.core.util.Health; + import org.apache.usergrid.persistence.core.util.StringUtils; import org.apache.usergrid.persistence.core.util.ValidationUtils; import org.apache.usergrid.persistence.index.*; -import org.apache.usergrid.persistence.index.ElasticSearchQueryBuilder.SearchRequestBuilderStrategyV2; +import org.apache.usergrid.persistence.index.elasticsearchquerybuilder.SearchRequestBuilderStrategyV2; import org.apache.usergrid.persistence.index.exceptions.IndexException; import org.apache.usergrid.persistence.index.migration.IndexDataVersions; import org.apache.usergrid.persistence.index.query.ParsedQuery; http://git-wip-us.apache.org/repos/asf/usergrid/blob/817d7ffb/stack/services/src/main/java/org/apache/usergrid/services/AbstractService.java ----------------------------------------------------------------------
