[usergrid] branch master updated (dd9e096 -> 7c1b93e)

[mrusso]

This is an automated email from the ASF dual-hosted git repository.

mrusso pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/usergrid.git.


from dd9e096  Merge pull request #619 from keyurkarnik/keyurkarnik_akka
 new c672c9c  Fixed akka cluster issue to support more than 2 regions
 new eaa9aba  Added config to restric Akka UV checks to local region Unique 
Value checks don't use Akka by default anymore Fixed USERGRID-1355
 new 7c1b93e  Merge pull request #621 from keyurkarnik/master

The 10943 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.


Summary of changes:

[usergrid] branch master updated (ac313ab -> dd9e096)

[mrusso]

This is an automated email from the ASF dual-hosted git repository.

mrusso pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/usergrid.git.


from ac313ab  Merge pull request #620 from keyurkarnik/keyurkarnik_export
 new ce8db89  Fixed akka cluster issue to support more than 2 regions
 new 6115b96  WIP: Attempting to reduce Akka chatter
 new dd9e096  Merge pull request #619 from keyurkarnik/keyurkarnik_akka

The 10940 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.


Summary of changes:
 .../actorsystem/ActorSystemManager.java|  8 
 .../actorsystem/ActorSystemManagerImpl.java| 14 ++-
 .../collection/uniquevalues/UniqueValueActor.java  | 47 +-
 .../collection/uniquevalues/UniqueValuesFig.java   | 17 +++-
 4 files changed, 74 insertions(+), 12 deletions(-)

[usergrid] branch master updated: Made entity export threads configurable Enables configuring number of threads for fetching entities, assets and entity members This fixes issue USERGRID-1356

[mrusso]

This is an automated email from the ASF dual-hosted git repository.

mrusso pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/usergrid.git


The following commit(s) were added to refs/heads/master by this push:
 new a88956c  Made entity export threads configurable Enables configuring 
number of threads for fetching entities, assets and entity members This fixes 
issue USERGRID-1356
 new ac313ab  Merge pull request #620 from keyurkarnik/keyurkarnik_export
a88956c is described below

commit a88956cc5d7aad082dcc4bf6a8020cd52c7ec10c
Author: Keyur Karnik 
AuthorDate: Tue Jan 29 14:37:33 2019 -0800

Made entity export threads configurable
Enables configuring number of threads for fetching entities,
assets and entity members
This fixes issue USERGRID-1356
---
 .../java/org/apache/usergrid/tools/Export.java | 91 +++---
 1 file changed, 81 insertions(+), 10 deletions(-)

diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/Export.java 
b/stack/tools/src/main/java/org/apache/usergrid/tools/Export.java
index b07d09d..9a67945 100644
--- a/stack/tools/src/main/java/org/apache/usergrid/tools/Export.java
+++ b/stack/tools/src/main/java/org/apache/usergrid/tools/Export.java
@@ -41,6 +41,7 @@ import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
 
 import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.MissingOptionException;
 import org.apache.commons.cli.Option;
 import org.apache.commons.cli.OptionBuilder;
 import org.apache.commons.cli.Options;
@@ -59,7 +60,6 @@ import org.apache.usergrid.persistence.SimpleEntityRef;
 import org.apache.usergrid.persistence.collection.EntityCollectionManager;
 import 
org.apache.usergrid.persistence.collection.EntityCollectionManagerFactory;
 import org.apache.usergrid.persistence.core.scope.ApplicationScope;
-import org.apache.usergrid.persistence.graph.Edge;
 import org.apache.usergrid.persistence.graph.GraphManager;
 import org.apache.usergrid.persistence.graph.SearchByEdgeType;
 import org.apache.usergrid.persistence.graph.impl.SimpleEdge;
@@ -78,6 +78,7 @@ import com.fasterxml.jackson.core.JsonFactory;
 import com.fasterxml.jackson.core.JsonGenerator;
 import com.google.common.base.Optional;
 import com.google.common.collect.BiMap;
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
 import com.google.gson.Gson;
 import com.google.gson.GsonBuilder;
 
@@ -89,7 +90,8 @@ import rx.schedulers.Schedulers;
 public class Export extends ExportingToolBase {
 
 static final Logger logger = LoggerFactory.getLogger( Export.class );
-public static final String LAST_ID = "lastId";
+   private static final String ENTITY_FETCHER_THREADS = 
"entityFetchThreads";
+   private static final String ENTITY_MEMBER_FETCHER_MULT = 
"entityThreadMult";
 
 
 @Autowired
@@ -100,11 +102,23 @@ public class Export extends ExportingToolBase {
 private AllEntityIdsObservable allEntityIdsObs;
 private SimpleEdge lastEdge = null;
 
+//number of threads for fetching entity contents. Each thread will handle 
a batch of 1000 entity ids
+private int entityFetcherThreads = 50;
+//after an individual entity is fetched, the entity members like assets, 
connections etc need to be fetched
+//depending on how heavy the assets/connections might be, we might need to 
multiply the factor so that more threads are allocated
+//for  pulling the members quickly without the queue backing up.
+private int entityMemberFetcherMultiplier = 1;
+
+
 //TODO : Add blocking queues for these executors where appropriate
-private ExecutorService orgAppCollParallelizer = 
Executors.newFixedThreadPool(3);
-private ExecutorService entityFetcher = Executors.newFixedThreadPool(10);
-   private ExecutorService enitityMemberFetcher = 
Executors.newFixedThreadPool(10);
-   private ExecutorService assetsFetcher = 
Executors.newFixedThreadPool(10);
+private ExecutorService orgAppCollParallelizer;
+
+//fetches the entity content
+private ExecutorService entityFetcher;
+//fetches the entity members like connections etc for a given entity
+   private ExecutorService entityMemberFetcher;
+   //fetches the assets for a given entity
+   private ExecutorService assetsFetcher;

 
 @Override
@@ -113,13 +127,70 @@ public class Export extends ExportingToolBase {
   
Options options = super.createOptions();

-   Option lastId = OptionBuilder.withArgName( LAST_ID ).hasArg()
-.withDescription( "Last Entity Id to resume from" ).create( 
LAST_ID );
-   options.addOption( lastId);
+   
+   Option entityFetcherThreads = OptionBuilder.withArgName( 
ENTITY_FETCHER_THREADS ).hasArg()
+.withDescription( "Number of threads to fetch entities in 
parallel (defa

[2/2] usergrid git commit: Merge commit 'refs/pull/618/head' of github.com:apache/usergrid

[mrusso]

Merge commit 'refs/pull/618/head' of github.com:apache/usergrid


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/76dc7a4b
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/76dc7a4b
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/76dc7a4b

Branch: refs/heads/master
Commit: 76dc7a4b8bcf655d589680dfb93134f398c597c6
Parents: 87f53a9 8b7aa27
Author: Michael Russo 
Authored: Sat Dec 29 11:15:44 2018 -0800
Committer: Michael Russo 
Committed: Sat Dec 29 11:15:44 2018 -0800

--
 .../rest/management/ManagementResource.java |   3 +
 .../usergrid/security/shiro/ShiroCache.java | 131 +++
 2 files changed, 79 insertions(+), 55 deletions(-)
--

[1/2] usergrid git commit: Fixed shiro cache to ensure 1:1 mapping between token and principal Before this, if a user had multiple sessions with different tokens, only one Principal was stored in the

[mrusso]

Repository: usergrid
Updated Branches:
  refs/heads/master 87f53a96f -> 76dc7a4b8


Fixed shiro cache to ensure 1:1 mapping between token and principal
Before this, if a user had multiple sessions with different tokens,
only one Principal was stored in the cache, with the first token. Now
every user session has a principal mapped to it in the cache


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/8b7aa27d
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/8b7aa27d
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/8b7aa27d

Branch: refs/heads/master
Commit: 8b7aa27d8d08d1009d3d6e83ab6a85b300d8a31b
Parents: abec1d9
Author: Keyur Karnik 
Authored: Tue Dec 18 04:05:32 2018 -0800
Committer: Keyur Karnik 
Committed: Wed Dec 19 17:30:39 2018 -0800

--
 .../rest/management/ManagementResource.java |   3 +
 .../usergrid/security/shiro/ShiroCache.java | 131 +++
 2 files changed, 79 insertions(+), 55 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/usergrid/blob/8b7aa27d/stack/rest/src/main/java/org/apache/usergrid/rest/management/ManagementResource.java
--
diff --git 
a/stack/rest/src/main/java/org/apache/usergrid/rest/management/ManagementResource.java
 
b/stack/rest/src/main/java/org/apache/usergrid/rest/management/ManagementResource.java
index 48aec78..2f67a1d 100644
--- 
a/stack/rest/src/main/java/org/apache/usergrid/rest/management/ManagementResource.java
+++ 
b/stack/rest/src/main/java/org/apache/usergrid/rest/management/ManagementResource.java
@@ -189,6 +189,9 @@ public class ManagementResource extends 
AbstractContextResource {
 final boolean ssoEnabled = 
Boolean.parseBoolean(properties.getProperty(USERGRID_EXTERNAL_SSO_ENABLED));
 long tokenTtl;
 
+//@TODO - This code takes the access token from the principal in the 
cache instead of using the sesssion token.
+//The token needs to be taken from the thread context instead to 
ensure that the correct token for the session is used
+  
 PrincipalIdentifier userPrincipal  = (PrincipalIdentifier) 
SecurityUtils.getSubject().getPrincipal();
 if ( userPrincipal != null && 
userPrincipal.getAccessTokenCredentials() != null ) {
 this.access_token = 
userPrincipal.getAccessTokenCredentials().getToken();

http://git-wip-us.apache.org/repos/asf/usergrid/blob/8b7aa27d/stack/services/src/main/java/org/apache/usergrid/security/shiro/ShiroCache.java
--
diff --git 
a/stack/services/src/main/java/org/apache/usergrid/security/shiro/ShiroCache.java
 
b/stack/services/src/main/java/org/apache/usergrid/security/shiro/ShiroCache.java
index 48062f9..d2a30d3 100644
--- 
a/stack/services/src/main/java/org/apache/usergrid/security/shiro/ShiroCache.java
+++ 
b/stack/services/src/main/java/org/apache/usergrid/security/shiro/ShiroCache.java
@@ -16,7 +16,11 @@
  */
 package org.apache.usergrid.security.shiro;
 
-import com.fasterxml.jackson.core.type.TypeReference;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Set;
+import java.util.StringJoiner;
+
 import org.apache.shiro.cache.Cache;
 import org.apache.shiro.cache.CacheException;
 import org.apache.shiro.subject.SimplePrincipalCollection;
@@ -24,14 +28,18 @@ import org.apache.usergrid.persistence.cache.CacheFactory;
 import org.apache.usergrid.persistence.cache.CacheScope;
 import org.apache.usergrid.persistence.cache.ScopedCache;
 import org.apache.usergrid.persistence.model.entity.SimpleId;
-import org.apache.usergrid.security.shiro.principals.*;
+import org.apache.usergrid.security.shiro.credentials.AccessTokenCredentials;
+import org.apache.usergrid.security.shiro.principals.AdminUserPrincipal;
+import org.apache.usergrid.security.shiro.principals.ApplicationGuestPrincipal;
+import org.apache.usergrid.security.shiro.principals.ApplicationPrincipal;
+import org.apache.usergrid.security.shiro.principals.ApplicationUserPrincipal;
+import org.apache.usergrid.security.shiro.principals.OrganizationPrincipal;
+import org.apache.usergrid.security.shiro.principals.PrincipalIdentifier;
 import org.apache.usergrid.security.shiro.utils.LocalShiroCache;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Set;
+import com.fasterxml.jackson.core.type.TypeReference;
 
 
 /**
@@ -58,9 +66,9 @@ public class ShiroCache implements Cache {
 if ( cacheTtl == 0 ) return null;
 
 V value;
-
+String ks = getKeyString(key);
 //check cache first
-value = (V) localShiroCache.get(getKeyString(key));
+value = (V) localShiroCache.get(ks);

[1/3] usergrid git commit: Minor updates that prevent deployment errors

[mrusso]

Repository: usergrid
Updated Branches:
  refs/heads/master 6af6f17a3 -> 87f53a96f


Minor updates that prevent deployment errors


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/07557785
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/07557785
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/07557785

Branch: refs/heads/master
Commit: 075577853f229b17cc8877c86ff02506a26d69e1
Parents: 0635958
Author: Joost van Ulden 
Authored: Mon Aug 27 20:07:01 2018 -0700
Committer: Joost van Ulden 
Committed: Mon Aug 27 20:07:01 2018 -0700

--
 deployment/aws/pom.xml|  7 ---
 .../aws/src/main/groovy/tag_instance.groovy   | 18 --
 2 files changed, 4 insertions(+), 21 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/usergrid/blob/07557785/deployment/aws/pom.xml
--
diff --git a/deployment/aws/pom.xml b/deployment/aws/pom.xml
index 88a423b..91ad738 100644
--- a/deployment/aws/pom.xml
+++ b/deployment/aws/pom.xml
@@ -30,7 +30,7 @@
 
 
 UTF-8
-ug-cloudformation
+fs-ug-cloudformation
 
 
 ../../stack/rest/target/ROOT.war
@@ -126,7 +126,7 @@
 
 net.java.dev.jets3t
 jets3t
-0.8.0
+0.9.3
 
 
 
@@ -218,6 +218,7 @@
 aws-java-sdk
 1.9.7
 
+
 
 groovy-all
 org.codehaus.groovy
@@ -229,7 +230,7 @@
 org.apache.usergrid
 usergrid-rest
 
-2.1.1-SNAPSHOT
+2.2.0-SNAPSHOT
 war
 
 

http://git-wip-us.apache.org/repos/asf/usergrid/blob/07557785/deployment/aws/src/main/groovy/tag_instance.groovy
--
diff --git a/deployment/aws/src/main/groovy/tag_instance.groovy 
b/deployment/aws/src/main/groovy/tag_instance.groovy
index 99d3288..e48881f 100644
--- a/deployment/aws/src/main/groovy/tag_instance.groovy
+++ b/deployment/aws/src/main/groovy/tag_instance.groovy
@@ -1,21 +1,3 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- *  contributor license agreements.  The ASF licenses this file to You
- * under the Apache License, Version 2.0 (the "License"); you may not
- * use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.  For additional information regarding
- * copyright in this work, please see the NOTICE file in the top level
- * directory of this distribution.
- */
-
 
 // 
 // tag_instance.groovy 

[2/3] usergrid git commit: PATCH - Updated Portal Dependencies

[mrusso]

PATCH - Updated Portal Dependencies

Portal will now build without errors.


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/4fd27f12
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/4fd27f12
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/4fd27f12

Branch: refs/heads/master
Commit: 4fd27f12e440639de829fffb3b3bc424616f0b25
Parents: 0755778
Author: Joost van Ulden 
Authored: Tue Aug 28 20:26:39 2018 -0700
Committer: Joost van Ulden 
Committed: Tue Aug 28 20:26:39 2018 -0700

--
 portal/Gruntfile.js |  2 +-
 portal/package.json | 51 +---
 2 files changed, 28 insertions(+), 25 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/usergrid/blob/4fd27f12/portal/Gruntfile.js
--
diff --git a/portal/Gruntfile.js b/portal/Gruntfile.js
index f0cdc94..9ea0c43 100644
--- a/portal/Gruntfile.js
+++ b/portal/Gruntfile.js
@@ -94,7 +94,7 @@ module.exports = function(grunt) {
   mangle: false,
   compress: false,
   beautify: true,
-  wrap: true
+  wrap: false
 },
 files: {
   'js/generated/usergrid-dev.js': [

http://git-wip-us.apache.org/repos/asf/usergrid/blob/4fd27f12/portal/package.json
--
diff --git a/portal/package.json b/portal/package.json
index 74a68ac..8b7588f 100644
--- a/portal/package.json
+++ b/portal/package.json
@@ -1,5 +1,5 @@
 {
-  "name": "usergrid", 
+  "name": "usergrid",
   "version": "2.0.19",
   "packageName": "appsvc-ui",
   "description": "full usergrid portal",
@@ -9,37 +9,40 @@
 "url": "https://github.com/usergrid/usergrid.git";
   },
   "devDependencies": {
-"chromedriver": "~2.10.0-1",
-"grunt": "~0.4.4",
-"grunt-cli": "~0.1.11",
-"grunt-contrib-cssmin": "~0.7.0",
-"grunt-contrib-uglify": "~0.2.7",
+"bower": "~1.2.8",
+"chromedriver": "^2.41.0",
+"grunt": ">=0.4.5 <1.0.0",
 "grunt-angular-templates": "~0.4.7",
-"grunt-contrib-watch": "~0.5.3",
-"grunt-contrib-connect": "~0.5.0",
+"grunt-bower-task": "^0.5.0",
+"grunt-cli": "^1.3.1",
+"grunt-contrib-clean": "~0.4.0",
+"grunt-contrib-compress": "^1.4.3",
 "grunt-contrib-concat": "~0.3.0",
-"karma-script-launcher": "~0.1.0",
+"grunt-contrib-connect": "^1.0.2",
+"grunt-contrib-copy": "~0.4.1",
+"grunt-contrib-cssmin": "~0.7.0",
+"grunt-contrib-uglify": "^3.4.0",
+"grunt-contrib-watch": "^1.1.0",
+"grunt-dom-munger": "~3.1.0",
+"grunt-istanbul": "^0.8.0",
+"grunt-karma": "2.0.0",
+"grunt-protractor-runner": "^5.0.0",
+"karma": "^3.0.0",
 "karma-chrome-launcher": "~0.1.1",
-"karma-html2js-preprocessor": "~0.1.0",
+"karma-coffee-preprocessor": "~0.1.1",
 "karma-firefox-launcher": "~0.1.2",
+"karma-html2js-preprocessor": "~0.1.0",
 "karma-jasmine": "~0.1.3",
-"karma-coffee-preprocessor": "~0.1.1",
-"requirejs": "~2.1.9",
+"karma-phantomjs-launcher": "^1.0.4",
 "karma-requirejs": "~0.2.0",
-"karma-phantomjs-launcher": "~0.1.1",
-"karma": "~0.10.8",
-"grunt-karma": "~0.6.2",
-"protractor": "~0.22.0",
-"grunt-protractor-runner": "~0.2.0",
-"grunt-contrib-copy": "~0.4.1",
-"grunt-contrib-compress": "~0.12.0",
-"grunt-contrib-clean": "~0.4.0",
-"grunt-dom-munger": "~3.1.0",
-"bower": "~1.2.8",
-"grunt-bower-task": "~0.4.0",
-"grunt-istanbul": "~0.2.5"
+"karma-script-launcher": "~0.1.0",
+"protractor": "^5.4.0",
+"requirejs": "~2.1.9"
   },
   "engines": {
 "node": ">=0.10.21"
+  },
+  "dependencies": {
+"grunt-jslint": "^1.1.15"
   }
 }

[3/3] usergrid git commit: Merge commit 'refs/pull/596/head' of github.com:apache/usergrid

[mrusso]

Merge commit 'refs/pull/596/head' of github.com:apache/usergrid


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/87f53a96
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/87f53a96
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/87f53a96

Branch: refs/heads/master
Commit: 87f53a96ff14d4ab6bb394b5e9150991e98bf6c0
Parents: 6af6f17 4fd27f1
Author: Michael Russo 
Authored: Sat Dec 29 11:10:00 2018 -0800
Committer: Michael Russo 
Committed: Sat Dec 29 11:10:00 2018 -0800

--
 portal/Gruntfile.js |  2 +-
 portal/package.json | 51 +---
 2 files changed, 28 insertions(+), 25 deletions(-)
--

[2/3] usergrid git commit: Merge pull request #1 from bursech/uniqueValueRepaireWork

[mrusso]

Merge pull request #1 from bursech/uniqueValueRepaireWork

Tool to perform read repair on cassandra and also to  repair entities wit…

Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/08f8dd50
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/08f8dd50
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/08f8dd50

Branch: refs/heads/master
Commit: 08f8dd500cb65991a8d102a39f68706f8e49a015
Parents: abec1d9 d12d82c
Author: keyurkarnik <42590775+keyurkar...@users.noreply.github.com>
Authored: Thu Dec 13 17:28:32 2018 -0800
Committer: GitHub 
Committed: Thu Dec 13 17:28:32 2018 -0800

--
 .../usergrid/tools/UniqueValueRepairer.java | 448 +++
 1 file changed, 448 insertions(+)
--

[2/2] usergrid git commit: Merge commit 'refs/pull/599/head' of github.com:apache/usergrid

[mrusso]

Merge commit 'refs/pull/599/head' of github.com:apache/usergrid


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/628ad012
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/628ad012
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/628ad012

Branch: refs/heads/master
Commit: 628ad01295547fd0f705b6549f6b8fa5d432128e
Parents: abec1d9 71c43df
Author: Michael Russo 
Authored: Sat Dec 29 11:07:25 2018 -0800
Committer: Michael Russo 
Committed: Sat Dec 29 11:07:25 2018 -0800

--
 stack/README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--

[1/3] usergrid git commit: Tool to perform read repair on cassandra and also to repair entities with missing unique value entry. Have options to dry run only to detect entities with missing unique val

[mrusso]

Repository: usergrid
Updated Branches:
  refs/heads/master 628ad0129 -> 6af6f17a3


Tool to perform read repair on cassandra and also to  repair entities with 
missing unique value entry.
Have options to dry run only to detect entities with missing unique value entry.


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/d12d82c2
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/d12d82c2
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/d12d82c2

Branch: refs/heads/master
Commit: d12d82c2b3f31d9d9b4625422f016486a257e245
Parents: abec1d9
Author: Chetan Burse 
Authored: Mon Nov 26 13:17:55 2018 -0800
Committer: Chetan Burse 
Committed: Thu Dec 13 17:20:53 2018 -0800

--
 .../usergrid/tools/UniqueValueRepairer.java | 448 +++
 1 file changed, 448 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/usergrid/blob/d12d82c2/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueValueRepairer.java
--
diff --git 
a/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueValueRepairer.java 
b/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueValueRepairer.java
new file mode 100644
index 000..04915be
--- /dev/null
+++ 
b/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueValueRepairer.java
@@ -0,0 +1,448 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.usergrid.tools;
+
+import java.io.UnsupportedEncodingException;
+import java.lang.reflect.Method;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.OptionBuilder;
+import org.apache.commons.cli.Options;
+import org.apache.usergrid.corepersistence.rx.impl.AllEntityIdsObservable;
+import org.apache.usergrid.corepersistence.rx.impl.EdgeScope;
+import org.apache.usergrid.corepersistence.util.CpNamingUtils;
+import org.apache.usergrid.management.OrganizationInfo;
+import org.apache.usergrid.persistence.Entity;
+import org.apache.usergrid.persistence.EntityManager;
+import org.apache.usergrid.persistence.Results;
+import org.apache.usergrid.persistence.collection.MvccEntity;
+import 
org.apache.usergrid.persistence.collection.serialization.MvccEntitySerializationStrategy;
+import org.apache.usergrid.persistence.collection.serialization.UniqueValue;
+import 
org.apache.usergrid.persistence.collection.serialization.UniqueValueSerializationStrategy;
+import org.apache.usergrid.persistence.collection.serialization.UniqueValueSet;
+import 
org.apache.usergrid.persistence.collection.serialization.impl.UniqueValueImpl;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.core.scope.ApplicationScopeImpl;
+import org.apache.usergrid.persistence.graph.impl.SimpleEdge;
+import org.apache.usergrid.persistence.model.entity.Id;
+import org.apache.usergrid.persistence.model.entity.SimpleId;
+import org.apache.usergrid.persistence.model.field.StringField;
+import org.apache.usergrid.utils.ConversionUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.datastax.driver.core.ConsistencyLevel;
+import com.datastax.driver.core.Session;
+import com.fasterxml.jackson.core.JsonFactory;
+import com.google.common.base.Optional;
+import com.google.common.collect.BiMap;
+
+import rx.Observable;
+import rx.observables.ConnectableObservable;
+import rx.schedulers.Schedulers;
+
+public class UniqueValueRepairer extends ExportingToolBase {
+
+   static final Logger logger = 
LoggerFactory.getLogger(UniqueValueRepairer.class);
+
+   JsonFactory jsonFactory = ne

[3/3] usergrid git commit: Merge commit 'refs/pull/617/head' of github.com:apache/usergrid

[mrusso]

Merge commit 'refs/pull/617/head' of github.com:apache/usergrid


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/6af6f17a
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/6af6f17a
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/6af6f17a

Branch: refs/heads/master
Commit: 6af6f17a333e9f0edd3ca9212164befea596f8e5
Parents: 628ad01 08f8dd5
Author: Michael Russo 
Authored: Sat Dec 29 11:08:03 2018 -0800
Committer: Michael Russo 
Committed: Sat Dec 29 11:08:03 2018 -0800

--
 .../usergrid/tools/UniqueValueRepairer.java | 448 +++
 1 file changed, 448 insertions(+)
--

[1/2] usergrid git commit: stack REDME link type fixed

[mrusso]

Repository: usergrid
Updated Branches:
  refs/heads/master abec1d9b6 -> 628ad0129


stack REDME link type fixed


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/71c43df3
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/71c43df3
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/71c43df3

Branch: refs/heads/master
Commit: 71c43df34f9d9cc409494b6c687a8bc51a64d11d
Parents: faafed8
Author: mddanishyusuf 
Authored: Wed Oct 3 23:12:08 2018 +0530
Committer: mddanishyusuf 
Committed: Wed Oct 3 23:12:08 2018 +0530

--
 stack/README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/usergrid/blob/71c43df3/stack/README.md
--
diff --git a/stack/README.md b/stack/README.md
index 268cb7e..6166da2 100644
--- a/stack/README.md
+++ b/stack/README.md
@@ -3,7 +3,7 @@
 A highly-scalable data platform for mobile applications.
 
 * **Documentation**: http://usergrid.apache.org/docs/
-* **Homepage**: http://http://usergrid.apache.org/
+* **Homepage**: http://usergrid.apache.org/
 
 
 ## Requirements

usergrid git commit: Revamped Export code to be more efficient Export tool now works in parallel for multiple collections Exports can be resumed from last exported edge Export is substantially faster

[mrusso]

Repository: usergrid
Updated Branches:
  refs/heads/master 5fc3c506a -> abec1d9b6


Revamped Export code to be more efficient
Export tool now works in parallel for multiple collections
Exports can be resumed from last exported edge
Export is substantially faster due to parallel processing
Connection fetching is optimzed
Export now creates multiple dirs for apps and collections


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/abec1d9b
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/abec1d9b
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/abec1d9b

Branch: refs/heads/master
Commit: abec1d9b60ce39941aee5b363bd5cf0f03e18e11
Parents: 5fc3c50
Author: Keyur Karnik 
Authored: Tue Nov 6 16:42:40 2018 -0800
Committer: Keyur Karnik 
Committed: Tue Nov 6 16:42:55 2018 -0800

--
 .../java/org/apache/usergrid/tools/Export.java  | 942 ---
 .../usergrid/tools/ExportingToolBase.java   | 151 ++-
 .../org/apache/usergrid/tools/ToolBase.java |  48 +-
 3 files changed, 803 insertions(+), 338 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/usergrid/blob/abec1d9b/stack/tools/src/main/java/org/apache/usergrid/tools/Export.java
--
diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/Export.java 
b/stack/tools/src/main/java/org/apache/usergrid/tools/Export.java
index 821d011..b07d09d 100644
--- a/stack/tools/src/main/java/org/apache/usergrid/tools/Export.java
+++ b/stack/tools/src/main/java/org/apache/usergrid/tools/Export.java
@@ -17,66 +17,201 @@
 package org.apache.usergrid.tools;
 
 
+import static 
org.apache.usergrid.management.AccountCreationProps.PROPERTIES_USERGRID_BINARY_UPLOADER;
+
+import java.io.BufferedOutputStream;
+import java.io.BufferedWriter;
 import java.io.File;
+import java.io.FileOutputStream;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
 import java.io.UnsupportedEncodingException;
+import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
 import java.util.Set;
 import java.util.UUID;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
 
-import com.fasterxml.jackson.core.JsonEncoding;
-import com.fasterxml.jackson.core.JsonFactory;
-import com.fasterxml.jackson.core.JsonGenerator;
-import org.apache.usergrid.persistence.*;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.OptionBuilder;
+import org.apache.commons.cli.Options;
+import org.apache.usergrid.corepersistence.ManagerCache;
+import org.apache.usergrid.corepersistence.export.ExportRequestBuilder;
+import org.apache.usergrid.corepersistence.export.ExportRequestBuilderImpl;
+import org.apache.usergrid.corepersistence.rx.impl.AllEntityIdsObservable;
+import org.apache.usergrid.corepersistence.rx.impl.EdgeScope;
+import org.apache.usergrid.corepersistence.util.CpNamingUtils;
 import org.apache.usergrid.management.OrganizationInfo;
 import org.apache.usergrid.management.UserInfo;
+import org.apache.usergrid.persistence.Entity;
+import org.apache.usergrid.persistence.EntityManager;
+import org.apache.usergrid.persistence.Results;
+import org.apache.usergrid.persistence.SimpleEntityRef;
+import org.apache.usergrid.persistence.collection.EntityCollectionManager;
+import 
org.apache.usergrid.persistence.collection.EntityCollectionManagerFactory;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.graph.Edge;
+import org.apache.usergrid.persistence.graph.GraphManager;
+import org.apache.usergrid.persistence.graph.SearchByEdgeType;
+import org.apache.usergrid.persistence.graph.impl.SimpleEdge;
+import org.apache.usergrid.persistence.graph.impl.SimpleSearchByEdgeType;
+import org.apache.usergrid.persistence.model.entity.Id;
+import org.apache.usergrid.persistence.model.entity.SimpleId;
+import org.apache.usergrid.services.assets.BinaryStoreFactory;
+import org.apache.usergrid.services.assets.data.BinaryStore;
 import org.apache.usergrid.tools.bean.ExportOrg;
-import org.apache.usergrid.utils.JsonUtils;
-
-import org.apache.commons.cli.CommandLine;
+import org.json.JSONObject;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
 
+import com.fasterxml.jackson.core.JsonFactory;
+import com.fasterxml.jackson.core.JsonGenerator;
+import com.google.common.base.Optional;
 import com.google.common.collect.BiMap;
+im

[07/16] usergrid git commit: fix NPE for missing names in direct queries and block requests based on direct query count before deduping

[mrusso]

fix NPE for missing names in direct queries and block requests based on direct 
query count before deduping


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/98a2f5d2
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/98a2f5d2
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/98a2f5d2

Branch: refs/heads/master
Commit: 98a2f5d23d4736219d23c637223a891c47216eed
Parents: 59e7a24
Author: Mike Dunker 
Authored: Thu Apr 12 18:03:47 2018 -0700
Committer: Keyur Karnik 
Committed: Tue Aug 28 16:41:44 2018 -0700

--
 .../pipeline/read/search/CandidateEntityFilter.java| 13 -
 .../persistence/index/impl/EsEntityIndexImpl.java  |  7 ---
 .../usergrid/persistence/index/query/ParsedQuery.java  |  9 +
 3 files changed, 21 insertions(+), 8 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/usergrid/blob/98a2f5d2/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/search/CandidateEntityFilter.java
--
diff --git 
a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/search/CandidateEntityFilter.java
 
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/search/CandidateEntityFilter.java
index 955b419..cf1984d 100644
--- 
a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/search/CandidateEntityFilter.java
+++ 
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/search/CandidateEntityFilter.java
@@ -289,13 +289,16 @@ public class CandidateEntityFilter extends 
AbstractFilter foundUUIDs = new HashSet<>();
 for (FilterResult candidateFilterResult : 
candidateResults) {
-UUID uuid = 
candidateFilterResult.getValue().getCandidateResult().getId().getUuid();
-if (!foundUUIDs.contains(uuid)) {
-dedupedCandidateResults.add(candidateFilterResult);
-foundUUIDs.add(uuid);
+Id id = 
candidateFilterResult.getValue().getCandidateResult().getId();
+if (id != null) {
+UUID uuid = id.getUuid();
+if (!foundUUIDs.contains(uuid)) {
+dedupedCandidateResults.add(candidateFilterResult);
+foundUUIDs.add(uuid);
+}
 }
 }
 }

http://git-wip-us.apache.org/repos/asf/usergrid/blob/98a2f5d2/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
--
diff --git 
a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
 
b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
index 6dfb2ae..82be6ec 100644
--- 
a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
+++ 
b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
@@ -456,14 +456,15 @@ public class EsEntityIndexImpl implements 
EntityIndex,VersionedData {
 throw new IllegalArgumentException("a null query string cannot be 
parsed");
 }
 
+if (parsedQuery.isDirectQuery() && 
parsedQuery.getDirectQueryItemCount() > indexFig.directQueryMaxItems()) {
+throw new 
TooManyDirectEntitiesException(parsedQuery.getDirectQueryItemCount(), 
indexFig.directQueryMaxItems());
+}
+
 final QueryVisitor visitor = visitParsedQuery(parsedQuery);
 
 List directIdentifiers = visitor.getDirectIdentifiers();
 if (directIdentifiers != null && directIdentifiers.size() > 0) {
 // this is a direct query
-if (directIdentifiers.size() > indexFig.directQueryMaxItems()) {
-throw new 
TooManyDirectEntitiesException(directIdentifiers.size(), 
indexFig.directQueryMaxItems());
-}
 return buildCandidateResultsForDirectQuery(directIdentifiers, 
parsedQuery, searchTypes);
 }
 

http://git-wip-us.apache.org/repos/asf/usergrid/blob/98a2f5d2/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/ParsedQuery.java
--
diff --git 
a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/ParsedQuery.java
 
b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/ParsedQuery.java
index 80ba6b1..4c0da5d 100644
--- 
a/sta

[09/16] usergrid git commit: fix deindex on update -- was deindexing current version along with old versions

[mrusso]

fix deindex on update -- was deindexing current version along with old versions


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/c98a5e92
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/c98a5e92
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/c98a5e92

Branch: refs/heads/master
Commit: c98a5e92d1f2bf2424624e6c1f76511a345f5d69
Parents: 1feb3db
Author: Mike Dunker 
Authored: Thu Mar 8 15:05:09 2018 -0800
Committer: Keyur Karnik 
Committed: Tue Aug 28 16:41:44 2018 -0700

--
 .../corepersistence/asyncevents/EventBuilderImpl.java| 8 
 1 file changed, 4 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/usergrid/blob/c98a5e92/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilderImpl.java
--
diff --git 
a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilderImpl.java
 
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilderImpl.java
index 1eb5e03..dbec084 100644
--- 
a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilderImpl.java
+++ 
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilderImpl.java
@@ -301,14 +301,14 @@ public class EventBuilderImpl implements EventBuilder {
 
 
 return indexService.deIndexOldVersions( applicationScope, entityId,
-getVersionsOlderThanOrEqualToMarked(ecm, entityId, markedVersion));
+getVersionsOlderThanMarked(ecm, entityId, markedVersion));
 
 
 }
 
 
-private List getVersionsOlderThanOrEqualToMarked(final 
EntityCollectionManager ecm,
-   final Id entityId, 
final UUID markedVersion ){
+private List getVersionsOlderThanMarked(final 
EntityCollectionManager ecm, final Id entityId,
+  final UUID markedVersion ){
 
 final List versions = new ArrayList<>();
 
@@ -317,7 +317,7 @@ public class EventBuilderImpl implements EventBuilder {
 ecm.getVersionsFromMaxToMin( entityId, markedVersion)
 .take(100)
 .forEach( mvccLogEntry -> {
-if ( mvccLogEntry.getVersion().timestamp() <= 
markedVersion.timestamp() ) {
+if ( mvccLogEntry.getVersion().timestamp() < 
markedVersion.timestamp() ) {
 versions.add(mvccLogEntry.getVersion());
 }
 

[04/16] usergrid git commit: bypass ES for push notification devices specified by UUID

[mrusso]

bypass ES for push notification devices specified by UUID


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/e8dc09bb
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/e8dc09bb
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/e8dc09bb

Branch: refs/heads/master
Commit: e8dc09bb3961ae91f5631b14344c7bcd62539263
Parents: 455bdde
Author: Mike Dunker 
Authored: Sat Feb 24 15:59:39 2018 -0800
Committer: Keyur Karnik 
Committed: Tue Aug 28 16:41:43 2018 -0700

--
 .../apache/usergrid/persistence/PathQuery.java  | 34 ++--
 1 file changed, 32 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/usergrid/blob/e8dc09bb/stack/core/src/main/java/org/apache/usergrid/persistence/PathQuery.java
--
diff --git 
a/stack/core/src/main/java/org/apache/usergrid/persistence/PathQuery.java 
b/stack/core/src/main/java/org/apache/usergrid/persistence/PathQuery.java
index c5833af..51b20c6 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/PathQuery.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/PathQuery.java
@@ -23,10 +23,14 @@ import java.util.UUID;
 
 import org.apache.usergrid.persistence.Query.Level;
 import org.apache.usergrid.utils.InflectionUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 
 public class PathQuery {
 
+private static final Logger logger = LoggerFactory.getLogger( 
PathQuery.class );
+
 private PathQuery source;
 private Query query;
 private UUID uuid;
@@ -122,10 +126,30 @@ public class PathQuery {
 
 EntityRef ref = new SimpleEntityRef(type,uuid);
 
-// if it's a single name identifier, just directly fetch that
-if ( !query.getQl().isPresent() && 
query.getSingleNameOrEmailIdentifier() != null){
+if ( !query.getQl().isPresent() && query.getSingleUuidIdentifier() != 
null) {
+
+// fetch entity by UUID directly
+UUID entityUUID = query.getSingleUuidIdentifier();
+
+if (logger.isTraceEnabled()) {
+logger.trace("getHeadResults(): identified path uuid: 
appid={}, collection={}, entityUUID={}",
+uuid.toString(), query.getCollection(), 
entityUUID.toString());
+}
+
+String entityType = 
InflectionUtils.singularize(query.getCollection());
+
+return em.getEntities(Collections.singletonList(entityUUID), 
entityType);
 
+} else if ( !query.getQl().isPresent() && 
query.getSingleNameOrEmailIdentifier() != null){
+
+// if it's a single name identifier, just directly fetch that
 String name = query.getSingleNameOrEmailIdentifier();
+
+if (logger.isTraceEnabled()) {
+logger.trace("getHeadResults(): identified path name/email: 
appid={}, collection={}, name={}",
+uuid.toString(), query.getCollection(), name);
+}
+
 String entityType = 
InflectionUtils.singularize(query.getCollection());
 
 // don't use unique index repair on read only logic
@@ -140,6 +164,12 @@ public class PathQuery {
 return em.getEntities(Collections.singletonList(entityId), 
entityType);
 }
 
+if (logger.isTraceEnabled()) {
+logger.trace("getHeadResults(): sending query to ES: appid={}, 
collection={}, query=[{}]",
+uuid.toString(), query.getCollection(),
+query.getQl().isPresent() ? query.getQl().get() : "NONE");
+}
+
 return ( query.getCollection() != null ) ?
em.searchCollection( ref, query.getCollection(), query ) :
em.searchTargetEntities(ref, query);

[01/16] usergrid git commit: exclude CollectionDeleteTest for now

[mrusso]

Repository: usergrid
Updated Branches:
  refs/heads/master faafed885 -> 5fc3c506a


exclude CollectionDeleteTest for now


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/35a22edd
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/35a22edd
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/35a22edd

Branch: refs/heads/master
Commit: 35a22edd3fe7e3356254f2770f2c22d8c7685789
Parents: a4f32fa
Author: Mike Dunker 
Authored: Tue Mar 6 12:19:35 2018 -0800
Committer: Keyur Karnik 
Committed: Tue Aug 28 16:41:43 2018 -0700

--
 .../java/org/apache/usergrid/persistence/CollectionDeleteTest.java | 2 ++
 1 file changed, 2 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/usergrid/blob/35a22edd/stack/core/src/test/java/org/apache/usergrid/persistence/CollectionDeleteTest.java
--
diff --git 
a/stack/core/src/test/java/org/apache/usergrid/persistence/CollectionDeleteTest.java
 
b/stack/core/src/test/java/org/apache/usergrid/persistence/CollectionDeleteTest.java
index 9e5598c..ae32a58 100644
--- 
a/stack/core/src/test/java/org/apache/usergrid/persistence/CollectionDeleteTest.java
+++ 
b/stack/core/src/test/java/org/apache/usergrid/persistence/CollectionDeleteTest.java
@@ -33,6 +33,7 @@ import org.apache.usergrid.persistence.model.entity.SimpleId;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
+import org.junit.Ignore;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -42,6 +43,7 @@ import static org.junit.Assert.*;
 
 
 @NotThreadSafe
+@Ignore("fix later")
 public class CollectionDeleteTest extends AbstractCoreIT {
 private static final Logger logger = LoggerFactory.getLogger( 
CollectionDeleteTest.class );
 

[14/16] usergrid git commit: Fixed queryindex test to match new operands

[mrusso]

Fixed queryindex test to match new operands


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/1c3fbe4f
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/1c3fbe4f
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/1c3fbe4f

Branch: refs/heads/master
Commit: 1c3fbe4fc77c96f279087298a48b5143a0d961b6
Parents: 0ac
Author: Keyur Karnik 
Authored: Wed Aug 29 01:16:51 2018 -0700
Committer: Keyur Karnik 
Committed: Wed Aug 29 01:16:51 2018 -0700

--
 .../usergrid/persistence/query/tree/GrammarTreeTest.java   | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/usergrid/blob/1c3fbe4f/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/query/tree/GrammarTreeTest.java
--
diff --git 
a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/query/tree/GrammarTreeTest.java
 
b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/query/tree/GrammarTreeTest.java
index 380057f..c459e43 100644
--- 
a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/query/tree/GrammarTreeTest.java
+++ 
b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/query/tree/GrammarTreeTest.java
@@ -681,9 +681,9 @@ public class GrammarTreeTest {
 }
 
 assertEquals(
-"NoViableAltException('!'@[1:1: Tokens : ( T__31 | T__32 | 
T__33 | T__34 | T__35 | T__36 | T__37 | "
-+ "T__38 | T__39 | T__40 | LT | LTE | EQ | GT | GTE | 
BOOLEAN | AND | OR | NOT | ASC | DESC |"
-+ " CONTAINS | WITHIN | OF | UUID | ID | LONG | FLOAT 
| STRING | WS );])",
+"NoViableAltException('!'@[1:1: Tokens : ( T__32 | T__33 | 
T__34 | T__35 | T__36 | T__37 | T__38 | T__39 | T__40 |"
+   +" T__41 | LT | LTE | EQ | GT | GTE | BOOLEAN | AND | OR | NOT 
| ASC | DESC | CONTAINS | WITHIN | OF | DIRECT |"
+   +" UUID | ID | LONG | FLOAT | STRING | WS );])",
 error );
 }
 }

[06/16] usergrid git commit: Add ability to query by name/uuid without using Elasticsearch

[mrusso]

Add ability to query by name/uuid without using Elasticsearch

example ql: "direct widget1,56d8fac2-39ef-11e8-b467-0ed5f89f718b,widget3"


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/57afad20
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/57afad20
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/57afad20

Branch: refs/heads/master
Commit: 57afad20a6c152550cc7f9fb3d35d445dbb90a25
Parents: 38c7313
Author: Mike Dunker 
Authored: Fri Apr 6 16:05:56 2018 -0700
Committer: Keyur Karnik 
Committed: Tue Aug 28 16:41:44 2018 -0700

--
 .../pipeline/PipelineContext.java   |  20 +++-
 .../search/AbstractElasticSearchFilter.java |   2 +-
 .../read/search/CandidateEntityFilter.java  |  86 ---
 .../index/query/tree/CpQueryFilter.g| 110 ---
 .../src/main/java/CpQueryFilter.tokens  |  67 +--
 .../persistence/index/CandidateResult.java  |  97 +++-
 .../persistence/index/CandidateResults.java |  12 ++
 .../usergrid/persistence/index/EntityIndex.java |  20 
 .../usergrid/persistence/index/IndexFig.java|   6 +
 .../TooManyDirectEntitiesException.java |  49 +
 .../index/impl/EsEntityIndexImpl.java   |  56 --
 .../persistence/index/impl/EsQueryVistor.java   |  58 +++---
 .../persistence/index/query/ParsedQuery.java|   5 +
 .../index/query/tree/DirectOperand.java |  82 ++
 .../persistence/index/query/tree/IdLiteral.java |  46 
 .../index/query/tree/QueryVisitor.java  |  18 ++-
 16 files changed, 616 insertions(+), 118 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/usergrid/blob/57afad20/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/PipelineContext.java
--
diff --git 
a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/PipelineContext.java
 
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/PipelineContext.java
index 88b5001..7aa614a 100644
--- 
a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/PipelineContext.java
+++ 
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/PipelineContext.java
@@ -28,6 +28,10 @@ import 
org.apache.usergrid.corepersistence.pipeline.cursor.ResponseCursor;
 import org.apache.usergrid.persistence.core.scope.ApplicationScope;
 
 import com.google.common.base.Optional;
+import org.apache.usergrid.persistence.index.query.ParsedQuery;
+import org.apache.usergrid.persistence.index.query.ParsedQueryBuilder;
+
+import static org.apache.usergrid.persistence.Query.MAX_LIMIT;
 
 
 /**
@@ -43,16 +47,24 @@ public class PipelineContext {
 // it can happen if ES was not updated or has yet to be updated.
 private final boolean keepStaleEntries;
 private String query;
+private ParsedQuery parsedQuery;
 
 
 public PipelineContext( final ApplicationScope applicationScope, final 
RequestCursor requestCursor, final int limit, final int id, boolean 
keepStaleEntries, String query ) {
 
 this.applicationScope = applicationScope;
-this.requestCursor = requestCursor;
-this.limit = limit;
 this.id = id;
 this.keepStaleEntries = keepStaleEntries;
 this.query = query;
+this.parsedQuery = ParsedQueryBuilder.build(query);
+if (parsedQuery != null && parsedQuery.isDirectQuery()) {
+// for direct query, use no limit or cursor
+this.limit = MAX_LIMIT + 1;
+this.requestCursor = new RequestCursor(Optional.absent());
+} else {
+this.limit = limit;
+this.requestCursor = requestCursor;
+}
 }
 
 
@@ -98,4 +110,8 @@ public class PipelineContext {
 return query;
 }
 
+public ParsedQuery getParsedQuery() {
+return parsedQuery;
+}
+
 }

http://git-wip-us.apache.org/repos/asf/usergrid/blob/57afad20/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/search/AbstractElasticSearchFilter.java
--
diff --git 
a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/search/AbstractElasticSearchFilter.java
 
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/search/AbstractElasticSearchFilter.java
index 7a46507..cc409b0 100644
--- 
a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/search/AbstractElasticSearchFilter.java
+++ 
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/search/AbstractElasticSearchFilter.java
@@ -129,7 +129,7 @@ public abstract class AbstractElasticSearchFilter ex

[13/16] usergrid git commit: return 400 when direct query has too many items

[mrusso]

return 400 when direct query has too many items


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/0ac9
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/0ac9
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/0ac9

Branch: refs/heads/master
Commit: 0ac90b9beb00037f83af9cd49044a83618d5
Parents: 98a2f5d
Author: Mike Dunker 
Authored: Thu Apr 12 18:27:31 2018 -0700
Committer: Keyur Karnik 
Committed: Tue Aug 28 16:48:05 2018 -0700

--
 .../TooManyDirectEntitiesExceptionMapper.java| 15 +++
 1 file changed, 15 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/usergrid/blob/0ac9/stack/rest/src/main/java/org/apache/usergrid/rest/exceptions/TooManyDirectEntitiesExceptionMapper.java
--
diff --git 
a/stack/rest/src/main/java/org/apache/usergrid/rest/exceptions/TooManyDirectEntitiesExceptionMapper.java
 
b/stack/rest/src/main/java/org/apache/usergrid/rest/exceptions/TooManyDirectEntitiesExceptionMapper.java
new file mode 100644
index 000..843086d
--- /dev/null
+++ 
b/stack/rest/src/main/java/org/apache/usergrid/rest/exceptions/TooManyDirectEntitiesExceptionMapper.java
@@ -0,0 +1,15 @@
+package org.apache.usergrid.rest.exceptions;
+
+
+import javax.ws.rs.core.Response;
+import javax.ws.rs.ext.Provider;
+import static javax.ws.rs.core.Response.Status.BAD_REQUEST;
+import 
org.apache.usergrid.persistence.index.exceptions.TooManyDirectEntitiesException;
+
+@Provider
+public class TooManyDirectEntitiesExceptionMapper extends 
AbstractExceptionMapper {
+@Override
+public Response toResponse( final TooManyDirectEntitiesException e ) {
+return toResponse( BAD_REQUEST, e );
+}
+}

[16/16] usergrid git commit: Fixed failing tests and bugs -Fixed issue where query was not being passed correctly for ql search on connections -Fixed NPE for direct query -Fixed tests that were failin

[mrusso]

Fixed failing tests and bugs
-Fixed issue where query was not being passed correctly for ql search on
connections
-Fixed NPE for direct query
-Fixed tests that were failing sometimes due to parallel execution
-Fixed CollectionDeleteTest so that it doesn not have to be ignored


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/5fc3c506
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/5fc3c506
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/5fc3c506

Branch: refs/heads/master
Commit: 5fc3c506a14ac28a702a2c146466b9f3907e4125
Parents: 749a97f
Author: Keyur Karnik 
Authored: Mon Sep 24 01:58:40 2018 -0700
Committer: Keyur Karnik 
Committed: Mon Sep 24 17:42:11 2018 -0700

--
 .../read/search/CandidateEntityFilter.java  |  4 +-
 .../service/ConnectionServiceImpl.java  |  7 ++-
 .../persistence/CollectionDeleteTest.java   | 62 ++--
 .../TooManyDirectEntitiesExceptionMapper.java   | 18 ++
 .../applications/queries/AndOrQueryTest.java|  3 +
 5 files changed, 59 insertions(+), 35 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/usergrid/blob/5fc3c506/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/search/CandidateEntityFilter.java
--
diff --git 
a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/search/CandidateEntityFilter.java
 
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/search/CandidateEntityFilter.java
index cf1984d..c0db02f 100644
--- 
a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/search/CandidateEntityFilter.java
+++ 
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/search/CandidateEntityFilter.java
@@ -27,6 +27,7 @@ import 
org.apache.usergrid.corepersistence.index.IndexLocationStrategyFactory;
 import org.apache.usergrid.persistence.Schema;
 import org.apache.usergrid.persistence.index.*;
 import org.apache.usergrid.persistence.index.impl.IndexProducer;
+import org.apache.usergrid.persistence.index.query.ParsedQuery;
 import org.apache.usergrid.persistence.model.entity.SimpleId;
 import org.apache.usergrid.persistence.model.field.DistanceField;
 import org.apache.usergrid.persistence.model.field.EntityObjectField;
@@ -103,7 +104,8 @@ public class CandidateEntityFilter extends 
AbstractFilter> searchIdSetObservable =

http://git-wip-us.apache.org/repos/asf/usergrid/blob/5fc3c506/stack/core/src/main/java/org/apache/usergrid/corepersistence/service/ConnectionServiceImpl.java
--
diff --git 
a/stack/core/src/main/java/org/apache/usergrid/corepersistence/service/ConnectionServiceImpl.java
 
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/service/ConnectionServiceImpl.java
index 259e4b9..c476f1f 100644
--- 
a/stack/core/src/main/java/org/apache/usergrid/corepersistence/service/ConnectionServiceImpl.java
+++ 
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/service/ConnectionServiceImpl.java
@@ -84,8 +84,11 @@ public class ConnectionServiceImpl implements 
ConnectionService {
 final Optional query = search.getQuery();
 
 final IdBuilder pipelineBuilder =
-pipelineBuilderFactory.create( search.getApplicationScope() 
).withCursor( search.getCursor() )
-  .withLimit( search.getLimit() ).fromId( 
search.getSourceNodeId() );
+pipelineBuilderFactory.create( search.getApplicationScope() )
+   .withCursor( search.getCursor() )
+   .withLimit( search.getLimit() )
+   .query(query)
+   .fromId( search.getSourceNodeId() );
 
 
 //we want to load all entities

http://git-wip-us.apache.org/repos/asf/usergrid/blob/5fc3c506/stack/core/src/test/java/org/apache/usergrid/persistence/CollectionDeleteTest.java
--
diff --git 
a/stack/core/src/test/java/org/apache/usergrid/persistence/CollectionDeleteTest.java
 
b/stack/core/src/test/java/org/apache/usergrid/persistence/CollectionDeleteTest.java
index ae32a58..c22a627 100644
--- 
a/stack/core/src/test/java/org/apache/usergrid/persistence/CollectionDeleteTest.java
+++ 
b/stack/core/src/test/java/org/apache/usergrid/persistence/CollectionDeleteTest.java
@@ -17,33 +17,31 @@
 package org.apache.usergrid.persistence;
 
 
-import com.codahale.metrics.MetricRegistry;
-import com.google.inject.Injector;
-import net.jcip.annotations.NotThreadSafe;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+

[05/16] usergrid git commit: fix NPE with push notifications and sendMessageToLocalRegion

[mrusso]

fix NPE with push notifications and sendMessageToLocalRegion


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/455bdde7
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/455bdde7
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/455bdde7

Branch: refs/heads/master
Commit: 455bdde7004f1f610f6ac2395298167ee9a3e7b2
Parents: 8913e43
Author: Mike Dunker 
Authored: Sat Feb 24 12:09:52 2018 -0800
Committer: Keyur Karnik 
Committed: Tue Aug 28 16:41:43 2018 -0700

--
 .../usergrid/persistence/queue/impl/SNSQueueManagerImpl.java   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/usergrid/blob/455bdde7/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/impl/SNSQueueManagerImpl.java
--
diff --git 
a/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/impl/SNSQueueManagerImpl.java
 
b/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/impl/SNSQueueManagerImpl.java
index de1671d..83b65de 100644
--- 
a/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/impl/SNSQueueManagerImpl.java
+++ 
b/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/impl/SNSQueueManagerImpl.java
@@ -682,7 +682,7 @@ public class SNSQueueManagerImpl implements 
LegacyQueueManager {
 
 @Override
 public  void sendMessageToLocalRegion(final T 
body, Boolean async) throws IOException {
-boolean sendAsync = async.booleanValue();
+boolean sendAsync = async == null || async.booleanValue();
 if (sendAsync) {
 sendMessageToLocalRegionAsync(body);
 } else {

[15/16] usergrid git commit: Merge pull request #1 from keyurkarnik/master

[mrusso]

Merge pull request #1 from keyurkarnik/master

Fixed failing tests and bugs

Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/749a97f8
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/749a97f8
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/749a97f8

Branch: refs/heads/master
Commit: 749a97f86e8f7470a73a90a0f6221a9dba9c4cf6
Parents: 1c3fbe4 faafed8
Author: keyurkarnik <42590775+keyurkar...@users.noreply.github.com>
Authored: Wed Sep 5 00:35:13 2018 -0700
Committer: GitHub 
Committed: Wed Sep 5 00:35:13 2018 -0700

--
 .../corepersistence/index/ReIndexService.java   |  2 +-
 .../index/ReIndexServiceImpl.java   | 22 
 .../usergrid/persistence/RebuildIndexTest.java  | 57 
 stack/pom.xml   |  2 +-
 .../apache/usergrid/rest/IndexResourceIT.java   | 45 +++-
 .../org/apache/usergrid/NewOrgAppAdminRule.java |  1 +
 .../management/OrganizationConfigIT.java|  4 +-
 .../management/export/ExportServiceIT.java  |  3 ++
 8 files changed, 98 insertions(+), 38 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/usergrid/blob/749a97f8/stack/pom.xml
--
diff --cc stack/pom.xml
index 41ce5bf,aa289d0..438a6fd
--- a/stack/pom.xml
+++ b/stack/pom.xml
@@@ -114,10 -114,10 +114,10 @@@
  4.12
  1.2.16
  
3.2.13.RELEASE
- 1.2.4
+ 1.3.2
  1.7.2
  1.9
 -7.0.64
 +7.0.84
  3.4
  1.4
  1.10.8

[08/16] usergrid git commit: Create AppDeleter tool to delete data for an application

[mrusso]

Create AppDeleter tool to delete data for an application


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/59e7a24e
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/59e7a24e
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/59e7a24e

Branch: refs/heads/master
Commit: 59e7a24e9ebd577bd892812bd36f2626fa011677
Parents: 57afad2
Author: Mike Dunker 
Authored: Fri Apr 6 16:08:41 2018 -0700
Committer: Keyur Karnik 
Committed: Tue Aug 28 16:41:44 2018 -0700

--
 .../corepersistence/CpEntityManager.java|   9 +-
 .../usergrid/persistence/EntityManager.java |   4 +
 .../usergrid/management/ManagementService.java  |   2 +
 .../cassandra/ManagementServiceImpl.java|  53 ++
 stack/tools/pom.xml |   6 +
 .../org/apache/usergrid/tools/AppDeleter.java   | 656 +++
 .../org/apache/usergrid/tools/ExportApp.java|   2 +-
 7 files changed, 730 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/usergrid/blob/59e7a24e/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
--
diff --git 
a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
 
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
index ed33201..0681b7a 100644
--- 
a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
+++ 
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
@@ -1246,8 +1246,15 @@ public class CpEntityManager implements EntityManager {
 
 @Override
 public Map getDictionaryAsMap( EntityRef entity, String 
dictionaryName ) throws Exception {
+return getDictionaryAsMap(entity, dictionaryName, true);
+}
+
+
+@Override
+public Map getDictionaryAsMap( EntityRef entity, String 
dictionaryName,
+   boolean forceVerification) 
throws Exception {
 
-entity = validate( entity );
+entity = validate( entity, forceVerification);
 
 Map dictionary = new LinkedHashMap();
 

http://git-wip-us.apache.org/repos/asf/usergrid/blob/59e7a24e/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java
--
diff --git 
a/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java 
b/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java
index c9752c3..c0e64a6 100644
--- 
a/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java
+++ 
b/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java
@@ -276,6 +276,10 @@ public interface EntityManager {
 public Map getDictionaryAsMap( EntityRef entityRef, String 
dictionaryName )
 throws Exception;
 
+public Map getDictionaryAsMap( EntityRef entityRef, String 
dictionaryName,
+   boolean forceVerification )
+throws Exception;
+
 public Object getDictionaryElementValue( EntityRef entityRef, String 
dictionaryName,
 String elementName ) throws Exception;
 

http://git-wip-us.apache.org/repos/asf/usergrid/blob/59e7a24e/stack/services/src/main/java/org/apache/usergrid/management/ManagementService.java
--
diff --git 
a/stack/services/src/main/java/org/apache/usergrid/management/ManagementService.java
 
b/stack/services/src/main/java/org/apache/usergrid/management/ManagementService.java
index 3d0c9fb..e0ab95e 100644
--- 
a/stack/services/src/main/java/org/apache/usergrid/management/ManagementService.java
+++ 
b/stack/services/src/main/java/org/apache/usergrid/management/ManagementService.java
@@ -110,6 +110,8 @@ public interface ManagementService {
 
UUID addApplicationToOrganization(UUID organizationId, Entity appInfo) 
throws Exception;
 
+   boolean deleteAdminUser( UUID userId ) throws Exception;
+
void deleteOrganizationApplication( UUID organizationId, UUID 
applicationId ) throws Exception;
 
void disableAdminUser( UUID userId ) throws Exception;

http://git-wip-us.apache.org/repos/asf/usergrid/blob/59e7a24e/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java
--
diff --git 
a/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java
 
b/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java
index ab93563..7e114e5 100644
--- 
a/stack/services/src/main/java/org/apache/usergri

[10/16] usergrid git commit: Several indexing fixes:

[mrusso]

Several indexing fixes:

1. remove possible duplicate entities from query results when using collection
   setting indexConsistency=latest
2. deindex all but latest candidate for an entity, even when using
   indexConsistency=latest
3. fix indexing bug when collection settings have been added but fields
   setting does not exist
4. by default deindex deleted entity even when indexing is off for the
   collection (this will clean up old index documents) - this is controlled by
   usergrid.entityManager.deindex_deleted_when_collection_indexing_off
   property.


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/1feb3dba
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/1feb3dba
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/1feb3dba

Branch: refs/heads/master
Commit: 1feb3dbab88145c9d5483b05bced03c2f6bafa79
Parents: 35a22ed
Author: Mike Dunker 
Authored: Thu Mar 8 00:32:39 2018 -0800
Committer: Keyur Karnik 
Committed: Tue Aug 28 16:41:44 2018 -0700

--
 .../corepersistence/CpEntityManager.java|   4 +-
 .../corepersistence/EntityManagerFig.java   |   4 +
 .../corepersistence/index/IndexServiceImpl.java |   5 +
 .../read/search/CandidateEntityFilter.java  | 109 +++
 .../queue/settings/IndexConsistency.java|   2 +-
 5 files changed, 99 insertions(+), 25 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/usergrid/blob/1feb3dba/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
--
diff --git 
a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
 
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
index 7a4c781..ed33201 100644
--- 
a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
+++ 
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
@@ -638,7 +638,9 @@ public class CpEntityManager implements EntityManager {
 
 Id entityId = new SimpleId( entityRef.getUuid(), entityRef.getType() );
 
-if ( !skipIndexingForType( entityId.getType() ) ) {
+// may still want to delete index entries even if indexing is turned 
off for new updates
+if ( entityManagerFig.deindexDeletedWhenCollectionIndexingOff() ||
+!skipIndexingForType( entityId.getType() ) ) {
 indexService.queueEntityDelete( applicationScope, entityId );
 }
 

http://git-wip-us.apache.org/repos/asf/usergrid/blob/1feb3dba/stack/core/src/main/java/org/apache/usergrid/corepersistence/EntityManagerFig.java
--
diff --git 
a/stack/core/src/main/java/org/apache/usergrid/corepersistence/EntityManagerFig.java
 
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/EntityManagerFig.java
index 3c8a53f..23cf1c3 100644
--- 
a/stack/core/src/main/java/org/apache/usergrid/corepersistence/EntityManagerFig.java
+++ 
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/EntityManagerFig.java
@@ -42,6 +42,10 @@ public interface EntityManagerFig extends GuicyFig {
 @Default( "false" )
 boolean getDeindexOnUpdate();
 
+@Key( 
"usergrid.entityManager.deindex_deleted_when_collection_indexing_off")
+@Default( "true" )
+boolean deindexDeletedWhenCollectionIndexingOff();
+
 /**
  * Comma-separated list of one or more Amazon regions to use if multiregion
  * is set to true.

http://git-wip-us.apache.org/repos/asf/usergrid/blob/1feb3dba/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java
--
diff --git 
a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java
 
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java
index 32470f6..b1d493e 100644
--- 
a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java
+++ 
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java
@@ -199,6 +199,11 @@ public class IndexServiceImpl implements IndexService {
 
 Object fields = jsonMapData.get("fields");
 
+// if "fields" field doesn't exist, should treat like fields=all
+if ( fields == null ) {
+return Optional.absent();
+}
+
 if ( fields != null && fields instanceof String && 
"all".equalsIgnoreCase(fields.toString())) {
 return Optional.absent();
 }

http://git-wip-us.apache.org/repos/asf/usergrid/blob/1feb3dba/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/search

[12/16] usergrid git commit: improvements to CollectionIterator tool for duplicate and orphaned connections

[mrusso]

improvements to CollectionIterator tool for duplicate and orphaned connections


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/0632ceff
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/0632ceff
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/0632ceff

Branch: refs/heads/master
Commit: 0632ceff5b40b3a23f1561589cb911c3ca45cb6b
Parents: c98a5e9
Author: Mike Dunker 
Authored: Mon Mar 12 12:19:56 2018 -0700
Committer: Keyur Karnik 
Committed: Tue Aug 28 16:41:44 2018 -0700

--
 .../persistence/index/utils/UUIDUtils.java  |  7 +-
 .../usergrid/tools/CollectionIterator.java  | 80 ++--
 .../org/apache/usergrid/tools/ToolBase.java | 26 +++
 3 files changed, 89 insertions(+), 24 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/usergrid/blob/0632ceff/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/utils/UUIDUtils.java
--
diff --git 
a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/utils/UUIDUtils.java
 
b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/utils/UUIDUtils.java
index bf8dd3c..1ba9f3c 100644
--- 
a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/utils/UUIDUtils.java
+++ 
b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/utils/UUIDUtils.java
@@ -303,8 +303,11 @@ public class UUIDUtils {
 if ( uuid == null ) {
 return 0;
 }
-long t = uuid.timestamp();
-return ( t - KCLOCK_OFFSET ) / KCLOCK_MULTIPLIER_L;
+return getUnixTimestampInMillisFromUUIDTimestamp(uuid.timestamp());
+}
+
+public static long getUnixTimestampInMillisFromUUIDTimestamp( long 
uuidTimestamp ) {
+return ( uuidTimestamp - KCLOCK_OFFSET ) / KCLOCK_MULTIPLIER_L;
 }
 
 

http://git-wip-us.apache.org/repos/asf/usergrid/blob/0632ceff/stack/tools/src/main/java/org/apache/usergrid/tools/CollectionIterator.java
--
diff --git 
a/stack/tools/src/main/java/org/apache/usergrid/tools/CollectionIterator.java 
b/stack/tools/src/main/java/org/apache/usergrid/tools/CollectionIterator.java
index 26b5f5f..bfe5edf 100644
--- 
a/stack/tools/src/main/java/org/apache/usergrid/tools/CollectionIterator.java
+++ 
b/stack/tools/src/main/java/org/apache/usergrid/tools/CollectionIterator.java
@@ -23,23 +23,16 @@ import java.util.*;
 
 import com.google.common.base.Optional;
 import com.netflix.astyanax.MutationBatch;
-import org.apache.usergrid.corepersistence.pipeline.read.ResultsPage;
-import org.apache.usergrid.corepersistence.results.EntityQueryExecutor;
-import org.apache.usergrid.corepersistence.results.IdQueryExecutor;
-import org.apache.usergrid.corepersistence.service.CollectionSearch;
 import org.apache.usergrid.corepersistence.service.CollectionService;
 import org.apache.usergrid.corepersistence.util.CpNamingUtils;
 import org.apache.usergrid.persistence.*;
 import org.apache.usergrid.persistence.core.scope.ApplicationScope;
 import org.apache.usergrid.persistence.core.scope.ApplicationScopeImpl;
 import org.apache.usergrid.persistence.graph.*;
-import org.apache.usergrid.persistence.graph.impl.SimpleEdge;
-import org.apache.usergrid.persistence.graph.impl.SimpleMarkedEdge;
 import org.apache.usergrid.persistence.graph.impl.SimpleSearchByEdgeType;
 import org.apache.usergrid.persistence.graph.serialization.EdgeSerialization;
 import org.apache.usergrid.persistence.index.utils.UUIDUtils;
 import org.apache.usergrid.persistence.model.entity.*;
-import org.apache.usergrid.persistence.model.entity.Entity;
 import org.apache.usergrid.persistence.schema.CollectionInfo;
 import org.apache.usergrid.utils.InflectionUtils;
 import static org.apache.commons.lang.StringUtils.isBlank;
@@ -63,7 +56,9 @@ public class CollectionIterator extends ToolBase {
 
 private static final String ENTITY_TYPE_ARG = "entityType";
 
-private static final String REMOVE_CONNECTIONS_ARG = "removeConnections";
+private static final String REMOVE_DUPLICATE_CONNECTIONS_ARG = 
"removeDuplicateConnections";
+
+private static final String REMOVE_ORPHAN_CONNECTIONS_ARG = 
"removeOrphanConnections";
 
 private static final String LATEST_TIMESTAMP_ARG = "latestTimestamp";
 
@@ -95,11 +90,17 @@ public class CollectionIterator extends ToolBase {
 
 options.addOption( collectionOption );
 
-Option removeConnectionsOption =
-
OptionBuilder.withArgName(REMOVE_CONNECTIONS_ARG).hasArg().isRequired( false 
).withDescription( "remove orphaned connections" )
-.create(REMOVE_CONNE

[11/16] usergrid git commit: Add password when soft deleting an application

[mrusso]

Add password when soft deleting an application

Will need the password when attempting to restore.


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/38c7313d
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/38c7313d
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/38c7313d

Branch: refs/heads/master
Commit: 38c7313dce04550e72e5bac434d831675ad5bf2d
Parents: 0632cef
Author: Mike Dunker 
Authored: Fri Apr 6 16:04:38 2018 -0700
Committer: Keyur Karnik 
Committed: Tue Aug 28 16:41:44 2018 -0700

--
 .../usergrid/corepersistence/CoreModule.java|  2 +
 .../ApplicationRestorePasswordService.java  | 25 +++
 .../ApplicationRestorePasswordServiceImpl.java  | 72 
 .../org/apache/usergrid/rest/ApiResponse.java   |  7 ++
 .../org/apache/usergrid/rest/RootResource.java  |  5 +-
 .../applications/ApplicationResource.java   | 37 ++
 6 files changed, 146 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/usergrid/blob/38c7313d/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
--
diff --git 
a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java 
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
index 909c073..841e978 100644
--- 
a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
+++ 
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
@@ -181,6 +181,8 @@ public class CoreModule extends AbstractModule {
 bind( ApplicationService.class ).to( ApplicationServiceImpl.class );
 
 bind( StatusService.class ).to( StatusServiceImpl.class );
+
+
bind(ApplicationRestorePasswordService.class).to(ApplicationRestorePasswordServiceImpl.class);
 }
 
 

http://git-wip-us.apache.org/repos/asf/usergrid/blob/38c7313d/stack/core/src/main/java/org/apache/usergrid/corepersistence/service/ApplicationRestorePasswordService.java
--
diff --git 
a/stack/core/src/main/java/org/apache/usergrid/corepersistence/service/ApplicationRestorePasswordService.java
 
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/service/ApplicationRestorePasswordService.java
new file mode 100644
index 000..99ef57d
--- /dev/null
+++ 
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/service/ApplicationRestorePasswordService.java
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.usergrid.corepersistence.service;
+
+import java.util.UUID;
+
+public interface ApplicationRestorePasswordService {
+String getApplicationRestorePassword(final UUID applicationId);
+void setApplicationRestorePassword(final UUID applicationId, final String 
restorePassword);
+void removeApplicationRestorePassword(final UUID applicationId);
+}

http://git-wip-us.apache.org/repos/asf/usergrid/blob/38c7313d/stack/core/src/main/java/org/apache/usergrid/corepersistence/service/ApplicationRestorePasswordServiceImpl.java
--
diff --git 
a/stack/core/src/main/java/org/apache/usergrid/corepersistence/service/ApplicationRestorePasswordServiceImpl.java
 
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/service/ApplicationRestorePasswordServiceImpl.java
new file mode 100644
index 000..e56bba7
--- /dev/null
+++ 
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/service/ApplicationRestorePasswordServiceImpl.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the Lic

[03/16] usergrid git commit: update to later tomcat version

[mrusso]

update to later tomcat version


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/ec7c6b48
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/ec7c6b48
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/ec7c6b48

Branch: refs/heads/master
Commit: ec7c6b48730ad233284dc536907dc73f6c01d8e1
Parents: e8dc09b
Author: Mike Dunker 
Authored: Thu Mar 1 07:41:32 2018 -0800
Committer: Keyur Karnik 
Committed: Tue Aug 28 16:41:43 2018 -0700

--
 stack/pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/usergrid/blob/ec7c6b48/stack/pom.xml
--
diff --git a/stack/pom.xml b/stack/pom.xml
index c98c72d..41ce5bf 100644
--- a/stack/pom.xml
+++ b/stack/pom.xml
@@ -117,7 +117,7 @@
 1.2.4
 1.7.2
 1.9
-7.0.64
+7.0.84
 3.4
 1.4
 1.10.8

[02/16] usergrid git commit: check for deleted entity candidates before checking for stale candidates

[mrusso]

check for deleted entity candidates before checking for stale candidates


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/a4f32fab
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/a4f32fab
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/a4f32fab

Branch: refs/heads/master
Commit: a4f32fab8b8b044d4a0d3f9436225a147d9ab6f8
Parents: ec7c6b4
Author: Mike Dunker 
Authored: Tue Mar 6 10:50:43 2018 -0800
Committer: Keyur Karnik 
Committed: Tue Aug 28 16:41:43 2018 -0700

--
 .../read/search/CandidateEntityFilter.java  | 29 ++--
 1 file changed, 14 insertions(+), 15 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/usergrid/blob/a4f32fab/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/search/CandidateEntityFilter.java
--
diff --git 
a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/search/CandidateEntityFilter.java
 
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/search/CandidateEntityFilter.java
index 20bcfe9..5dcbd27 100644
--- 
a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/search/CandidateEntityFilter.java
+++ 
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/search/CandidateEntityFilter.java
@@ -317,7 +317,20 @@ public class CandidateEntityFilter extends 
AbstractFilter 0 ) {
 
Date candidateTimeStamp = UUIDTimeStampToDate(candidateVersion);
@@ -344,20 +357,6 @@ public class CandidateEntityFilter extends 
AbstractFilter 0 ) {

usergrid git commit: Fix NPE for push notification processing.

[mrusso]

Repository: usergrid
Updated Branches:
  refs/heads/master bd2385d76 -> 063595805


Fix NPE for push notification processing.


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/06359580
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/06359580
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/06359580

Branch: refs/heads/master
Commit: 063595805cbf3efad6b961a9170bdf4953aa2e9d
Parents: bd2385d
Author: Michael Russo 
Authored: Sat Feb 24 11:44:06 2018 -0800
Committer: Michael Russo 
Committed: Sat Feb 24 11:44:06 2018 -0800

--
 .../services/notifications/impl/ApplicationQueueManagerImpl.java | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/usergrid/blob/06359580/stack/services/src/main/java/org/apache/usergrid/services/notifications/impl/ApplicationQueueManagerImpl.java
--
diff --git 
a/stack/services/src/main/java/org/apache/usergrid/services/notifications/impl/ApplicationQueueManagerImpl.java
 
b/stack/services/src/main/java/org/apache/usergrid/services/notifications/impl/ApplicationQueueManagerImpl.java
index b7afc2e..fcf63a8 100644
--- 
a/stack/services/src/main/java/org/apache/usergrid/services/notifications/impl/ApplicationQueueManagerImpl.java
+++ 
b/stack/services/src/main/java/org/apache/usergrid/services/notifications/impl/ApplicationQueueManagerImpl.java
@@ -356,11 +356,11 @@ public class ApplicationQueueManagerImpl implements 
ApplicationQueueManager {
 if(logger.isTraceEnabled()) {
 logger.trace("Queueing notification message 
for device: {}", message.get().getDeviceId());
 }
-qm.sendMessageToLocalRegion( message.get() , null 
);
+qm.sendMessageToLocalRegion( message.get(), 
Boolean.TRUE );
 queueMeter.mark();
 }
 
-} catch (IOException e) {
+} catch (Exception e) {
 
 if(message.isPresent()){
 logger.error("Unable to queue notification for 
notification UUID {} and device UUID {} ",

usergrid git commit: Add entity version to EntityVersionAudit csv file.

[mrusso]

Repository: usergrid
Updated Branches:
  refs/heads/master c8282f04b -> 598ed29a1


Add entity version to EntityVersionAudit csv file.


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/598ed29a
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/598ed29a
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/598ed29a

Branch: refs/heads/master
Commit: 598ed29a107e6af2905e708d5b9f1df27bb10679
Parents: c8282f0
Author: Michael Russo 
Authored: Thu Oct 26 12:32:20 2017 -0700
Committer: Michael Russo 
Committed: Thu Oct 26 12:32:20 2017 -0700

--
 .../java/org/apache/usergrid/tools/EntityVersionAudit.java  | 9 +
 1 file changed, 5 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/usergrid/blob/598ed29a/stack/tools/src/main/java/org/apache/usergrid/tools/EntityVersionAudit.java
--
diff --git 
a/stack/tools/src/main/java/org/apache/usergrid/tools/EntityVersionAudit.java 
b/stack/tools/src/main/java/org/apache/usergrid/tools/EntityVersionAudit.java
index c9f4860..ee064bc 100644
--- 
a/stack/tools/src/main/java/org/apache/usergrid/tools/EntityVersionAudit.java
+++ 
b/stack/tools/src/main/java/org/apache/usergrid/tools/EntityVersionAudit.java
@@ -189,7 +189,7 @@ public class EntityVersionAudit extends ToolBase {
 final Writer versionAuditWriter = new BufferedWriter(new 
OutputStreamWriter(new FileOutputStream("entity_version_audit.txt"), "utf-8"));
 final Writer versionAggWriter = new BufferedWriter(new 
OutputStreamWriter(new FileOutputStream("entity_version_agg.txt"), "utf-8"));
 
-
versionAuditWriter.write("collection,entityUUID,cassandraTimestamp,elasticsearchTimestamp,indexDelayMillis,existsInElasticsearch\n");
+
versionAuditWriter.write("collection,entityUUID,entityVersion,cassandraTimestamp,elasticsearchTimestamp,indexDelayMillis,existsInElasticsearch\n");
 versionAuditWriter.flush();
 
 final EsProvider esProvider = injector.getInstance(EsProvider.class);
@@ -239,9 +239,10 @@ public class EntityVersionAudit extends ToolBase {
 String csvLine =
 collectionName + "," +
 uuid + "," +
-uuidTimestamp + "," +
-indexTimestamp + "," +
-diff + "," +
+mvccLogEntry.getVersion() + "," +
+uuidTimestamp + "," +
+indexTimestamp + "," +
+diff + "," +
 exists;
 
 //final String url = 
"/"+readAlias+"/entity/"+URLEncoder.encode(esDocId, "UTF-8");

usergrid git commit: Add tool for auditing versions of entities in Elasticsearch.

[mrusso]

Repository: usergrid
Updated Branches:
  refs/heads/master 893a04ddc -> c8282f04b


Add tool for auditing versions of entities in Elasticsearch.


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/c8282f04
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/c8282f04
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/c8282f04

Branch: refs/heads/master
Commit: c8282f04b2dbe602cdc28d0330cbf4168b4ce6ee
Parents: 893a04d
Author: Michael Russo 
Authored: Thu Oct 26 10:34:12 2017 -0700
Committer: Michael Russo 
Committed: Thu Oct 26 10:34:12 2017 -0700

--
 .../usergrid/tools/EntityVersionAudit.java  | 275 +++
 1 file changed, 275 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/usergrid/blob/c8282f04/stack/tools/src/main/java/org/apache/usergrid/tools/EntityVersionAudit.java
--
diff --git 
a/stack/tools/src/main/java/org/apache/usergrid/tools/EntityVersionAudit.java 
b/stack/tools/src/main/java/org/apache/usergrid/tools/EntityVersionAudit.java
new file mode 100644
index 000..c9f4860
--- /dev/null
+++ 
b/stack/tools/src/main/java/org/apache/usergrid/tools/EntityVersionAudit.java
@@ -0,0 +1,275 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.usergrid.tools;
+
+
+import com.google.common.base.Optional;
+import com.netflix.astyanax.MutationBatch;
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.OptionBuilder;
+import org.apache.commons.cli.Options;
+import org.apache.usergrid.corepersistence.index.IndexLocationStrategyFactory;
+import org.apache.usergrid.corepersistence.service.CollectionService;
+import org.apache.usergrid.corepersistence.util.CpNamingUtils;
+import org.apache.usergrid.persistence.EntityManager;
+import org.apache.usergrid.persistence.EntityRef;
+import org.apache.usergrid.persistence.Query;
+import org.apache.usergrid.persistence.SimpleEntityRef;
+import org.apache.usergrid.persistence.collection.EntityCollectionManager;
+import 
org.apache.usergrid.persistence.collection.EntityCollectionManagerFactory;
+import org.apache.usergrid.persistence.collection.MvccLogEntry;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.core.scope.ApplicationScopeImpl;
+import org.apache.usergrid.persistence.graph.GraphManager;
+import org.apache.usergrid.persistence.graph.GraphManagerFactory;
+import org.apache.usergrid.persistence.graph.SearchByEdgeType;
+import org.apache.usergrid.persistence.graph.impl.SimpleEdge;
+import org.apache.usergrid.persistence.graph.impl.SimpleSearchByEdgeType;
+import org.apache.usergrid.persistence.graph.serialization.EdgeSerialization;
+import org.apache.usergrid.persistence.index.IndexLocationStrategy;
+import org.apache.usergrid.persistence.index.SearchEdge;
+import org.apache.usergrid.persistence.index.impl.EsProvider;
+import org.apache.usergrid.persistence.index.utils.UUIDUtils;
+import org.apache.usergrid.persistence.model.entity.Id;
+import org.apache.usergrid.persistence.model.entity.SimpleId;
+import org.apache.usergrid.persistence.schema.CollectionInfo;
+import org.apache.usergrid.utils.InflectionUtils;
+import org.elasticsearch.action.get.GetRequest;
+import org.elasticsearch.action.get.GetResponse;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import rx.Observable;
+
+import java.io.*;
+import java.net.URLEncoder;
+import java.text.DateFormat;
+import java.text.SimpleDateFormat;
+import java.util.*;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import static org.apache.commons.lang.StringUtils.isBlank;
+import static 
org.apache.usergrid.corepersistence.util.CpNamingUtils.createSearchEdgeFromSource;
+import static org.apache.usergrid.persistence.Schema.getDefaultSchema;
+import static 
org.apache.usergrid.persistence.index.impl.IndexingUtils.createIndexDocId;
+
+
+public class EntityVersionAudit extends ToolBase {
+
+/*
+
+   

usergrid git commit: Initial commit for exposing collection re-index to non sysadmin users. [Forced Update!]

[mrusso]

Repository: usergrid
Updated Branches:
  refs/heads/expose-reindex ec8e5c7ca -> 230e99501 (forced update)


Initial commit for exposing collection re-index to non sysadmin users.


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/230e9950
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/230e9950
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/230e9950

Branch: refs/heads/expose-reindex
Commit: 230e9950152583dc4e6697c4348df650333e3b18
Parents: 4a65910
Author: Michael Russo 
Authored: Wed Jun 28 11:07:15 2017 -0700
Committer: Michael Russo 
Committed: Wed Oct 4 21:57:12 2017 -0700

--
 .../corepersistence/index/ReIndexService.java   |  33 +-
 .../index/ReIndexServiceImpl.java   | 106 +--
 .../rest/applications/CollectionResource.java   |  40 ---
 .../usergrid/rest/system/IndexResource.java |  78 --
 4 files changed, 199 insertions(+), 58 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/usergrid/blob/230e9950/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexService.java
--
diff --git 
a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexService.java
 
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexService.java
index b9238e5..d37f117 100644
--- 
a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexService.java
+++ 
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexService.java
@@ -20,6 +20,8 @@
 package org.apache.usergrid.corepersistence.index;
 
 
+import org.apache.usergrid.utils.StringUtils;
+
 /**
  * An interface for re-indexing all entities in an application
  */
@@ -47,6 +49,13 @@ public interface ReIndexService {
  */
 ReIndexStatus getStatus( final String jobId );
 
+/**
+ * Get the status of a collection job
+ * @param collectionName The collectionName for the rebuild index
+ * @return
+ */
+ReIndexStatus getStatusForCollection( final String appIdString, final 
String collectionName );
+
 
 /**
  * The response when requesting a re-index operation
@@ -56,14 +65,27 @@ public interface ReIndexService {
 final Status status;
 final long numberProcessed;
 final long lastUpdated;
+final String collectionName;
 
 
 public ReIndexStatus( final String jobId, final Status status, final 
long numberProcessed,
-  final long lastUpdated ) {
-this.jobId = jobId;
+  final long lastUpdated, final String 
collectionName ) {
+
+if(StringUtils.isNotEmpty(jobId)){
+this.jobId = jobId;
+}else {
+this.jobId = "";
+}
+
 this.status = status;
 this.numberProcessed = numberProcessed;
 this.lastUpdated = lastUpdated;
+
+if(StringUtils.isNotEmpty(collectionName)){
+this.collectionName = collectionName;
+}else {
+this.collectionName = "";
+}
 }
 
 
@@ -74,6 +96,13 @@ public interface ReIndexService {
 return jobId;
 }
 
+/**
+ * Get the jobId used to resume this operation
+ */
+public String getCollectionName() {
+return collectionName;
+}
+
 
 /**
  * Get the last updated time, as a long

http://git-wip-us.apache.org/repos/asf/usergrid/blob/230e9950/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexServiceImpl.java
--
diff --git 
a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexServiceImpl.java
 
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexServiceImpl.java
index 05602fc..d4fb249 100644
--- 
a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexServiceImpl.java
+++ 
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexServiceImpl.java
@@ -75,6 +75,7 @@ public class ReIndexServiceImpl implements ReIndexService {
 private static final String MAP_COUNT_KEY = "count";
 private static final String MAP_STATUS_KEY = "status";
 private static final String MAP_UPDATED_KEY = "lastUpdated";
+private static final String MAP_SEPARATOR = "|||";
 
 
 private final AllApplicationsObservable allApplicationsObservable;
@@ -140,7 +141,9 @@ public class ReIndexServiceImpl implements ReIndexService {
 
 // create an observable that loads a batch to be indexed
 
-if(reIndexRequestBuilder.getCollectionName().isPresent()) {
+fin

[14/27] usergrid git commit: Converts de-indexing of edges and entities to identify documents in the index to delete based on data in Cassandra vs. resource intensive queries to the index. Fixes issue

[mrusso]

http://git-wip-us.apache.org/repos/asf/usergrid/blob/c201f1f6/stack/websocket/pom.xml
--
diff --git a/stack/websocket/pom.xml b/stack/websocket/pom.xml
index af5ed56..72a5c9d 100644
--- a/stack/websocket/pom.xml
+++ b/stack/websocket/pom.xml
@@ -70,7 +70,7 @@
 ${basedir}/src/test/conf
   
   always
-  -Xmx${ug.heapmax} -Xms${ug.heapmin} -Dfile.encoding=UTF-8 
-Dsun.jnu.encoding=UTF-8 
-javaagent:${settings.localRepository}/org/jacoco/org.jacoco.agent/${jacoco.version}/org.jacoco.agent-${jacoco.version}-runtime.jar=destfile=${project.build.directory}/jacoco.exec
 
-javaagent:${settings.localRepository}/com/github/stephenc/jamm/0.2.5/jamm-0.2.5.jar
 ${ug.argline}
+  -Xmx${ug.heapmax} -Xms${ug.heapmin} -Dfile.encoding=UTF-8 
-Dsun.jnu.encoding=UTF-8 
-javaagent:${settings.localRepository}/org/jacoco/org.jacoco.agent/${jacoco.version}/org.jacoco.agent-${jacoco.version}-runtime.jar=destfile=${project.build.directory}/jacoco.exec
 
-javaagent:${settings.localRepository}/com/github/jbellis/jamm/0.3.0/jamm-0.3.0.jar
 ${ug.argline}
 
   
 

[26/27] usergrid git commit: code review changes

[mrusso]

code review changes


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/4a659106
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/4a659106
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/4a659106

Branch: refs/heads/expose-reindex
Commit: 4a659106eadefc83cce93f47d6b5090f65b8c1e3
Parents: 9a15108
Author: Peter Johnson 
Authored: Wed Sep 27 15:10:56 2017 -0700
Committer: Peter Johnson 
Committed: Wed Sep 27 15:10:56 2017 -0700

--
 .../apache/usergrid/corepersistence/CpEntityManager.java |  4 ++--
 .../corepersistence/asyncevents/EventBuilderImpl.java|  1 -
 .../persistence/queue/impl/SNSQueueManagerImpl.java  | 11 +++
 3 files changed, 9 insertions(+), 7 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/usergrid/blob/4a659106/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
--
diff --git 
a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
 
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
index 1dc4a89..3f3794e 100644
--- 
a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
+++ 
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
@@ -2854,7 +2854,7 @@ public class CpEntityManager implements EntityManager {
 
 // add to and index in collection of the application
 if ( !is_application) {
-updateIndexForEniity(eType, entity, timestamp);
+updateIndexForEntity(eType, entity, timestamp);
 }
 
 //write to our types map
@@ -2864,7 +2864,7 @@ public class CpEntityManager implements EntityManager {
 return entity;
 }
 
-private  void updateIndexForEniity(String eType, A 
entity,  long timestamp) throws Exception {
+private  void updateIndexForEntity(String eType, A 
entity, long timestamp) throws Exception {
 String collectionName = Schema.defaultCollectionName( eType );
 CpRelationManager cpr = ( CpRelationManager ) getRelationManager( 
getApplication() );
 cpr.addToCollection( collectionName, entity );

http://git-wip-us.apache.org/repos/asf/usergrid/blob/4a659106/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilderImpl.java
--
diff --git 
a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilderImpl.java
 
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilderImpl.java
index 203d32a..1eb5e03 100644
--- 
a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilderImpl.java
+++ 
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilderImpl.java
@@ -21,7 +21,6 @@ package org.apache.usergrid.corepersistence.asyncevents;
 
 
 import java.util.ArrayList;
-import java.util.Collections;
 import java.util.List;
 import java.util.UUID;
 

http://git-wip-us.apache.org/repos/asf/usergrid/blob/4a659106/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/impl/SNSQueueManagerImpl.java
--
diff --git 
a/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/impl/SNSQueueManagerImpl.java
 
b/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/impl/SNSQueueManagerImpl.java
index 775a64b..bc9be57 100644
--- 
a/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/impl/SNSQueueManagerImpl.java
+++ 
b/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/impl/SNSQueueManagerImpl.java
@@ -84,6 +84,8 @@ public class SNSQueueManagerImpl implements 
LegacyQueueManager {
 private static final int MIN_VISIBILITY_TIMEOUT = 1; //seconds
 private static final String DEAD_LETTER_QUEUE_SUFFIX = "_dead";
 
+private static final String FAILED_TO_SEND_MESSAGE = "FAILED INDEX 
REQUEST: Failed to send message to SNS Queue, sending asynchronously. 
Message:[{}] URL:[{}] Error:[{}]";
+
 static {
 
 /**
@@ -573,7 +575,7 @@ public class SNSQueueManagerImpl implements 
LegacyQueueManager {
 }
 } catch (Exception e) {
 if (logger.isErrorEnabled()) {
-logger.error("Failed to send this message {} to SNS queue at 
{}", stringBody, topicArn);
+logger.error(FAILED_TO_SEND_MESSAGE,  stringBody, topicArn, e);
 }
 sendMessageToAllRegionsAsync(body);
 }
@@ -591,7 +593,7 @@ public class SNSQueueManagerImpl implements 
LegacyQueueManager {
 
 final Str

[02/27] usergrid git commit: change test collection names so no race condition conflicts

[mrusso]

change test collection names so no race condition conflicts


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/7d9cc895
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/7d9cc895
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/7d9cc895

Branch: refs/heads/expose-reindex
Commit: 7d9cc89599508d74fdc7e59aec5ab980e55a871a
Parents: f22c6c4
Author: Mike Dunker 
Authored: Wed Aug 16 22:28:54 2017 -0700
Committer: Mike Dunker 
Committed: Wed Aug 16 22:28:54 2017 -0700

--
 .../applications/queries/AndOrQueryTest.java| 111 +++
 .../queries/BadGrammarQueryTest.java|  12 +-
 .../rest/applications/queries/OrderByTest.java  | 137 ++-
 .../queries/SelectMappingsQueryTest.java|  53 +++
 4 files changed, 160 insertions(+), 153 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/usergrid/blob/7d9cc895/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/AndOrQueryTest.java
--
diff --git 
a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/AndOrQueryTest.java
 
b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/AndOrQueryTest.java
index 4bdd3fc..d49460c 100644
--- 
a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/AndOrQueryTest.java
+++ 
b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/AndOrQueryTest.java
@@ -47,20 +47,20 @@ public class AndOrQueryTest extends QueryTestBase {
 @Test
 public void queryAndInclusive() throws IOException {
 int numOfEntities = 20;
-String collectionName = "activities";
+String collectionName = "apples";
 // create our test entities
 generateTestEntities(numOfEntities, collectionName);
 // Query where madeup = true (the last half) and the last quarter of 
entries
 QueryParameters params = new QueryParameters()
 .setQuery("select * where madeup = true AND ordinal >= " + 
(numOfEntities - numOfEntities / 4));
-Collection activities = 
this.app().collection("activities").get(params);
+Collection coll = this.app().collection(collectionName).get(params);
 // results should have madeup = true and ordinal 15-19
-assertEquals(numOfEntities / 4, 
activities.getResponse().getEntityCount());
+assertEquals(numOfEntities / 4, coll.getResponse().getEntityCount());
 // loop though entities that were returned, and test against the 
ordinals and values we are
 // expecting, starting with the last entity and decrementing
 int index = 19;
-while (activities.hasNext()) {
-Entity activity = activities.next();
+while (coll.hasNext()) {
+Entity activity = coll.next();
 // ensure the 'madeup' property is set to true
 
assertTrue(Boolean.parseBoolean(activity.get("madeup").toString()));
 // make sure the correct ordinal properties are returned
@@ -71,15 +71,15 @@ public class AndOrQueryTest extends QueryTestBase {
 @Test
 public void someTestProp() throws IOException {
 int numOfEntities = 20;
-String collectionName = "activities";
+String collectionName = "bananas";
 // create our test entities
 generateTestEntities(numOfEntities, collectionName);
 // Query where madeup = true (the last half) and the last quarter of 
entries
 QueryParameters params = new QueryParameters()
 .setQuery("where sometestprop = 'testprop'");
-Collection activities = 
this.app().collection("activities").get(params);
+Collection coll = this.app().collection(collectionName).get(params);
 // results should have madeup = true and ordinal 15-19
-assertEquals(10, activities.getResponse().getEntityCount());
+assertEquals(10, coll.getResponse().getEntityCount());
 
 
 }
@@ -87,15 +87,15 @@ public class AndOrQueryTest extends QueryTestBase {
 @Test
 public void someTestPropPartialContains() throws IOException {
 int numOfEntities = 20;
-String collectionName = "activities";
+String collectionName = "cantaloupes";
 // create our test entities
 generateTestEntities(numOfEntities, collectionName);
 // Query where madeup = true (the last half) and the last quarter of 
entries
 QueryParameters params = new QueryParameters()
 .setQuery("where sometestprop contains 'test*'");
-Collection activities = 
this.app().collection("activities").get(params);
+Collection coll = this.app().collection(collectionName).get(params);
 // results should have madeup = true and ordinal 15-

[15/27] usergrid git commit: Converts de-indexing of edges and entities to identify documents in the index to delete based on data in Cassandra vs. resource intensive queries to the index. Fixes issue

[mrusso]

Converts de-indexing of edges and entities to identify documents in the index 
to delete based on data in Cassandra vs. resource intensive queries to the 
index. Fixes issue where nodes were not actually getting deleted from graph as 
marked edges were being filtered out during the delete process itself. Update 
to a newer vs. of jamm (used for jvm memory management in the test framework).


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/c201f1f6
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/c201f1f6
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/c201f1f6

Branch: refs/heads/expose-reindex
Commit: c201f1f67aa0f373958c3886c893979215cba88f
Parents: 570e1ab
Author: Michael Russo 
Authored: Sat Jul 8 17:10:25 2017 -0700
Committer: Mike Dunker 
Committed: Fri Aug 18 17:00:43 2017 -0700

--
 .../corepersistence/EntityManagerFig.java   |   2 +-
 .../asyncevents/AsyncEventServiceImpl.java  |  23 +--
 .../asyncevents/EventBuilder.java   |  15 +-
 .../asyncevents/EventBuilderImpl.java   | 172 ++-
 .../corepersistence/index/IndexService.java |  16 +-
 .../corepersistence/index/IndexServiceImpl.java |  56 ++
 .../read/traverse/AbstractReadGraphFilter.java  |  37 +---
 .../AbstractReadReverseGraphFilter.java |  36 +---
 .../org/apache/usergrid/persistence/Query.java  |  17 ++
 .../rx/EdgesFromSourceObservableIT.java |   2 +-
 .../rx/EdgesToTargetObservableIT.java   |   4 +-
 .../core/src/test/resources/project.properties  |   2 +-
 .../persistence/graph/GraphManager.java |   2 +-
 .../graph/impl/GraphManagerImpl.java|  21 +--
 .../graph/impl/SimpleSearchByEdgeType.java  |   9 +
 .../graph/impl/stage/NodeDeleteListener.java|   6 +-
 .../impl/stage/NodeDeleteListenerImpl.java  |  64 +++
 .../graph/serialization/EdgesObservable.java|   6 +-
 .../impl/EdgeMetadataSerializationV2Impl.java   |   1 -
 .../serialization/impl/EdgesObservableImpl.java |  22 +--
 .../impl/TargetIdObservableImpl.java|   4 +-
 .../impl/migration/EdgeDataMigrationImpl.java   |   2 +-
 .../graph/impl/NodeDeleteListenerTest.java  |   8 +-
 .../usergrid/persistence/index/EntityIndex.java |  12 +-
 .../index/impl/EsEntityIndexImpl.java   |  63 ---
 .../persistence/index/query/Identifier.java |   3 +-
 stack/pom.xml   |   8 +-
 stack/rest/pom.xml  |   4 +-
 .../rest/src/test/resources/project.properties  |   2 +-
 stack/services/pom.xml  |   4 +-
 .../src/test/resources/project.properties   |   2 +-
 stack/test-utils/pom.xml|   4 +-
 .../src/test/resources/project.properties   |   2 +-
 stack/tools/pom.xml |   2 +-
 stack/websocket/pom.xml |   2 +-
 35 files changed, 286 insertions(+), 349 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/usergrid/blob/c201f1f6/stack/core/src/main/java/org/apache/usergrid/corepersistence/EntityManagerFig.java
--
diff --git 
a/stack/core/src/main/java/org/apache/usergrid/corepersistence/EntityManagerFig.java
 
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/EntityManagerFig.java
index 872ffbb..46c7a1d 100644
--- 
a/stack/core/src/main/java/org/apache/usergrid/corepersistence/EntityManagerFig.java
+++ 
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/EntityManagerFig.java
@@ -39,7 +39,7 @@ public interface EntityManagerFig extends GuicyFig {
 int sleep();
 
 @Key( "usergrid.entityManager.enable_deindex_on_update" )
-@Default( "true" )
+@Default( "false" )
 boolean getDeindexOnUpdate();
 
 /**

http://git-wip-us.apache.org/repos/asf/usergrid/blob/c201f1f6/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventServiceImpl.java
--
diff --git 
a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventServiceImpl.java
 
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventServiceImpl.java
index 0e55e9b..79a80c0 100644
--- 
a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventServiceImpl.java
+++ 
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventServiceImpl.java
@@ -640,8 +640,7 @@ public class AsyncEventServiceImpl implements 
AsyncEventService {
 }
 
 // default this observable's return to empty index operation message 
if nothing is emitted
-return eventBuilder.buildDeleteEdge(applicationScope, edge)
-

[12/27] usergrid git commit: test changes and additional logging

[mrusso]

test changes and additional logging


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/39ec4f2b
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/39ec4f2b
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/39ec4f2b

Branch: refs/heads/expose-reindex
Commit: 39ec4f2bb895e180f493349fb14e62ac9178e5ee
Parents: 400365c
Author: Mike Dunker 
Authored: Fri Aug 18 09:46:50 2017 -0700
Committer: Mike Dunker 
Committed: Fri Aug 18 09:46:50 2017 -0700

--
 .../corepersistence/CpEntityManagerFactory.java |  2 +-
 .../asyncevents/AsyncEventServiceImpl.java  |  2 ++
 .../apache/usergrid/persistence/RebuildIndexTest.java   | 12 ++--
 .../queues/impl/QueueSerializationImpl.java |  2 +-
 .../apache/usergrid/services/ActivitiesServiceIT.java   |  9 ++---
 .../apache/usergrid/services/ServiceInvocationIT.java   |  3 +++
 6 files changed, 19 insertions(+), 11 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/usergrid/blob/39ec4f2b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
--
diff --git 
a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
 
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
index cec7258..bad5b2c 100644
--- 
a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
+++ 
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
@@ -487,7 +487,7 @@ public class CpEntityManagerFactory implements 
EntityManagerFactory, Application
 // evict app Id from cache
 applicationIdCache.evictAppId(appName);
 
-logger.info("Initialized application {}", appName);
+logger.info("Initialized application {}, uuid {}", appName, 
appInfo.getUuid().toString());
 return appInfo;
 }
 

http://git-wip-us.apache.org/repos/asf/usergrid/blob/39ec4f2b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventServiceImpl.java
--
diff --git 
a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventServiceImpl.java
 
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventServiceImpl.java
index 530cf7d..9501ad3 100644
--- 
a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventServiceImpl.java
+++ 
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventServiceImpl.java
@@ -881,6 +881,8 @@ public class AsyncEventServiceImpl implements 
AsyncEventService {
 final int utilityCount = indexProcessorFig.getWorkerCountUtility();
 final int indexDeadCount = 
indexProcessorFig.getWorkerCountDeadLetter();
 final int utilityDeadCount = 
indexProcessorFig.getWorkerCountUtilityDeadLetter();
+logger.info("Starting queue workers for indexing: index={} indexDLQ={} 
utility={} utilityDLQ={}", indexCount, indexDeadCount,
+utilityCount, utilityDeadCount);
 
 for (int i = 0; i < indexCount; i++) {
 startWorker(QUEUE_NAME);

http://git-wip-us.apache.org/repos/asf/usergrid/blob/39ec4f2b/stack/core/src/test/java/org/apache/usergrid/persistence/RebuildIndexTest.java
--
diff --git 
a/stack/core/src/test/java/org/apache/usergrid/persistence/RebuildIndexTest.java
 
b/stack/core/src/test/java/org/apache/usergrid/persistence/RebuildIndexTest.java
index a7759de..d2bff37 100644
--- 
a/stack/core/src/test/java/org/apache/usergrid/persistence/RebuildIndexTest.java
+++ 
b/stack/core/src/test/java/org/apache/usergrid/persistence/RebuildIndexTest.java
@@ -77,7 +77,7 @@ public class RebuildIndexTest extends AbstractCoreIT {
 @Test( timeout = 12 )
 public void rebuildOneCollectionIndex() throws Exception {
 
-logger.info( "Started rebuildIndex()" );
+logger.info( "Started rebuildOneCollectionIndex()" );
 
 String rand = RandomStringUtils.randomAlphanumeric( 5 );
 final UUID appId = setup.createApplication( "org_" + rand, "app_" + 
rand );
@@ -172,7 +172,7 @@ public class RebuildIndexTest extends AbstractCoreIT {
 }
 
 
-@Test( timeout = 12 )
+@Test( timeout = 24 )
 public void rebuildIndex() throws Exception {
 
 logger.info( "Started rebuildIndex()" );
@@ -234,7 +234,7 @@ public class RebuildIndexTest extends AbstractCoreIT {
 }
 
 logger.info( "Created {} entities", ENTITIES_TO_INDEX );
-app.waitForQueueDrainAndRefreshIndex(15000);
+app.waitForQueueDrainAndRefresh

[20/27] usergrid git commit: Change collection clear from PUT to POST.

[mrusso]

Change collection clear from PUT to POST.


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/c8605211
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/c8605211
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/c8605211

Branch: refs/heads/expose-reindex
Commit: c8605211020c53639b5db24075577c272b9d6895
Parents: 3d3ba49
Author: Mike Dunker 
Authored: Thu Aug 31 17:45:38 2017 -0700
Committer: Mike Dunker 
Committed: Thu Aug 31 17:45:38 2017 -0700

--
 .../org/apache/usergrid/rest/applications/CollectionResource.java  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/usergrid/blob/c8605211/stack/rest/src/main/java/org/apache/usergrid/rest/applications/CollectionResource.java
--
diff --git 
a/stack/rest/src/main/java/org/apache/usergrid/rest/applications/CollectionResource.java
 
b/stack/rest/src/main/java/org/apache/usergrid/rest/applications/CollectionResource.java
index c9174c1..9c09806 100644
--- 
a/stack/rest/src/main/java/org/apache/usergrid/rest/applications/CollectionResource.java
+++ 
b/stack/rest/src/main/java/org/apache/usergrid/rest/applications/CollectionResource.java
@@ -196,7 +196,7 @@ public class CollectionResource extends ServiceResource {
 }
 
 
-@PUT
+@POST
 @Path("{itemName}/_clear")
 @Produces({MediaType.APPLICATION_JSON, "application/javascript"})
 @RequireApplicationAccess

[03/27] usergrid git commit: add a little more wait time

[mrusso]

add a little more wait time


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/36e28a87
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/36e28a87
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/36e28a87

Branch: refs/heads/expose-reindex
Commit: 36e28a8704bd129be9c78b16fb4f0b77e97381df
Parents: 7d9cc89
Author: Mike Dunker 
Authored: Wed Aug 16 22:30:34 2017 -0700
Committer: Mike Dunker 
Committed: Wed Aug 16 22:30:34 2017 -0700

--
 .../apache/usergrid/rest/applications/queries/QueryTestBase.java   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/usergrid/blob/36e28a87/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/QueryTestBase.java
--
diff --git 
a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/QueryTestBase.java
 
b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/QueryTestBase.java
index 0adafef..1eec9ae 100644
--- 
a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/QueryTestBase.java
+++ 
b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/QueryTestBase.java
@@ -65,7 +65,7 @@ public class QueryTestBase  extends AbstractRestIT {
 logger.info(entities[i].entrySet().toString());
 }
 //refresh the index so that they are immediately searchable
-this.waitForQueueDrainAndRefreshIndex();
+this.waitForQueueDrainAndRefreshIndex(1000);
 
 return entities;
 }

[09/27] usergrid git commit: change test collection names so no race condition conflicts

[mrusso]

change test collection names so no race condition conflicts


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/d06dbc95
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/d06dbc95
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/d06dbc95

Branch: refs/heads/expose-reindex
Commit: d06dbc95a5e41a9cebb20f5859860dd74b6ae9ac
Parents: 4b916c4
Author: Mike Dunker 
Authored: Wed Aug 16 22:28:54 2017 -0700
Committer: Mike Dunker 
Committed: Thu Aug 17 11:59:47 2017 -0700

--
 .../applications/queries/AndOrQueryTest.java| 111 +++
 .../queries/BadGrammarQueryTest.java|  12 +-
 .../rest/applications/queries/OrderByTest.java  | 137 ++-
 .../queries/SelectMappingsQueryTest.java|  53 +++
 4 files changed, 160 insertions(+), 153 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/usergrid/blob/d06dbc95/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/AndOrQueryTest.java
--
diff --git 
a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/AndOrQueryTest.java
 
b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/AndOrQueryTest.java
index 4bdd3fc..d49460c 100644
--- 
a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/AndOrQueryTest.java
+++ 
b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/AndOrQueryTest.java
@@ -47,20 +47,20 @@ public class AndOrQueryTest extends QueryTestBase {
 @Test
 public void queryAndInclusive() throws IOException {
 int numOfEntities = 20;
-String collectionName = "activities";
+String collectionName = "apples";
 // create our test entities
 generateTestEntities(numOfEntities, collectionName);
 // Query where madeup = true (the last half) and the last quarter of 
entries
 QueryParameters params = new QueryParameters()
 .setQuery("select * where madeup = true AND ordinal >= " + 
(numOfEntities - numOfEntities / 4));
-Collection activities = 
this.app().collection("activities").get(params);
+Collection coll = this.app().collection(collectionName).get(params);
 // results should have madeup = true and ordinal 15-19
-assertEquals(numOfEntities / 4, 
activities.getResponse().getEntityCount());
+assertEquals(numOfEntities / 4, coll.getResponse().getEntityCount());
 // loop though entities that were returned, and test against the 
ordinals and values we are
 // expecting, starting with the last entity and decrementing
 int index = 19;
-while (activities.hasNext()) {
-Entity activity = activities.next();
+while (coll.hasNext()) {
+Entity activity = coll.next();
 // ensure the 'madeup' property is set to true
 
assertTrue(Boolean.parseBoolean(activity.get("madeup").toString()));
 // make sure the correct ordinal properties are returned
@@ -71,15 +71,15 @@ public class AndOrQueryTest extends QueryTestBase {
 @Test
 public void someTestProp() throws IOException {
 int numOfEntities = 20;
-String collectionName = "activities";
+String collectionName = "bananas";
 // create our test entities
 generateTestEntities(numOfEntities, collectionName);
 // Query where madeup = true (the last half) and the last quarter of 
entries
 QueryParameters params = new QueryParameters()
 .setQuery("where sometestprop = 'testprop'");
-Collection activities = 
this.app().collection("activities").get(params);
+Collection coll = this.app().collection(collectionName).get(params);
 // results should have madeup = true and ordinal 15-19
-assertEquals(10, activities.getResponse().getEntityCount());
+assertEquals(10, coll.getResponse().getEntityCount());
 
 
 }
@@ -87,15 +87,15 @@ public class AndOrQueryTest extends QueryTestBase {
 @Test
 public void someTestPropPartialContains() throws IOException {
 int numOfEntities = 20;
-String collectionName = "activities";
+String collectionName = "cantaloupes";
 // create our test entities
 generateTestEntities(numOfEntities, collectionName);
 // Query where madeup = true (the last half) and the last quarter of 
entries
 QueryParameters params = new QueryParameters()
 .setQuery("where sometestprop contains 'test*'");
-Collection activities = 
this.app().collection("activities").get(params);
+Collection coll = this.app().collection(collectionName).get(params);
 // results should have madeup = true and ordinal 15-

[08/27] usergrid git commit: more cassandra.timeout increases for tests

[mrusso]

more cassandra.timeout increases for tests


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/4b916c4d
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/4b916c4d
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/4b916c4d

Branch: refs/heads/expose-reindex
Commit: 4b916c4d39fa0018fbe01f8e0489b0f5d083e41e
Parents: 70de6fd
Author: Mike Dunker 
Authored: Thu Aug 17 11:57:53 2017 -0700
Committer: Mike Dunker 
Committed: Thu Aug 17 11:57:53 2017 -0700

--
 stack/core/src/test/resources/usergrid-custom-test.properties  | 1 +
 .../collection/src/test/resources/dynamic-test.properties  | 1 +
 .../collection/src/test/resources/usergrid-CHOP.properties | 1 +
 .../collection/src/test/resources/usergrid-UNIT.properties | 1 +
 .../collection/src/test/resources/usergrid.properties  | 1 +
 .../common/src/test/resources/usergrid-UNIT.properties | 2 +-
 .../graph/src/test/resources/usergrid-AWS.properties   | 1 +
 .../graph/src/test/resources/usergrid-CHOP.properties  | 1 +
 .../graph/src/test/resources/usergrid-UNIT.properties  | 1 +
 stack/corepersistence/graph/src/test/resources/usergrid.properties | 1 +
 .../queryindex/src/test/resources/dynamic-test.properties  | 1 +
 .../queryindex/src/test/resources/usergrid-CHOP.properties | 1 +
 .../queryindex/src/test/resources/usergrid-UNIT.properties | 1 +
 stack/corepersistence/queue/src/test/resources/qakka.properties| 1 +
 stack/rest/src/test/resources/usergrid-custom-test.properties  | 1 +
 15 files changed, 15 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/usergrid/blob/4b916c4d/stack/core/src/test/resources/usergrid-custom-test.properties
--
diff --git a/stack/core/src/test/resources/usergrid-custom-test.properties 
b/stack/core/src/test/resources/usergrid-custom-test.properties
index 8f9058d..e32b146 100644
--- a/stack/core/src/test/resources/usergrid-custom-test.properties
+++ b/stack/core/src/test/resources/usergrid-custom-test.properties
@@ -16,6 +16,7 @@
 # with ug.heapmax=5000m and ug.heapmin=3000m (set in Maven settings.xml)
 #cassandra.connections=30
 cassandra.timeout.pool=2
+cassandra.timeout=2
 
 
 #Not a good number for real systems.  Write shards should be 2x cluster size 
from our tests

http://git-wip-us.apache.org/repos/asf/usergrid/blob/4b916c4d/stack/corepersistence/collection/src/test/resources/dynamic-test.properties
--
diff --git 
a/stack/corepersistence/collection/src/test/resources/dynamic-test.properties 
b/stack/corepersistence/collection/src/test/resources/dynamic-test.properties
index ca1a51c..59b8d31 100644
--- 
a/stack/corepersistence/collection/src/test/resources/dynamic-test.properties
+++ 
b/stack/corepersistence/collection/src/test/resources/dynamic-test.properties
@@ -2,6 +2,7 @@
 # safe dynamic property defaults for our testing via IDE or Maven
 cassandra.connections=30
 cassandra.timeout.pool=3
+cassandra.timeout=3
 cassandra.port=9160
 cassandra.hosts=localhost
 cassandra.cluster_name=Usergrid

http://git-wip-us.apache.org/repos/asf/usergrid/blob/4b916c4d/stack/corepersistence/collection/src/test/resources/usergrid-CHOP.properties
--
diff --git 
a/stack/corepersistence/collection/src/test/resources/usergrid-CHOP.properties 
b/stack/corepersistence/collection/src/test/resources/usergrid-CHOP.properties
index c82bf83..9e95663 100644
--- 
a/stack/corepersistence/collection/src/test/resources/usergrid-CHOP.properties
+++ 
b/stack/corepersistence/collection/src/test/resources/usergrid-CHOP.properties
@@ -20,6 +20,7 @@
 
 cassandra.connections=30
 cassandra.timeout.pool=3
+cassandra.timeout=3
 cassandra.port=9160
 
 # a comma delimited private IP address list to your chop cassandra cluster

http://git-wip-us.apache.org/repos/asf/usergrid/blob/4b916c4d/stack/corepersistence/collection/src/test/resources/usergrid-UNIT.properties
--
diff --git 
a/stack/corepersistence/collection/src/test/resources/usergrid-UNIT.properties 
b/stack/corepersistence/collection/src/test/resources/usergrid-UNIT.properties
index 82bea14..0253e6e 100644
--- 
a/stack/corepersistence/collection/src/test/resources/usergrid-UNIT.properties
+++ 
b/stack/corepersistence/collection/src/test/resources/usergrid-UNIT.properties
@@ -21,6 +21,7 @@
 # purposely setting connections lower to slow down the activity on the 
cassandra server locally
 cassandra.connections=5
 cassandra.timeout.pool=6
+cassandra.timeout=6

[13/27] usergrid git commit: move deletes to new delete queue -- read repair will fix attempts to access deleted entities and connections, so indexing and collection deletes can proceed more slowly th

[mrusso]

move deletes to new delete queue -- read repair will fix attempts to access 
deleted entities and connections, so indexing and collection deletes can 
proceed more slowly than other types of changes


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/570e1ab4
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/570e1ab4
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/570e1ab4

Branch: refs/heads/expose-reindex
Commit: 570e1ab4e2d4d356756fedee6b7ed5a1bd2d8e93
Parents: 39ec4f2
Author: Mike Dunker 
Authored: Fri Aug 18 15:12:56 2017 -0700
Committer: Mike Dunker 
Committed: Fri Aug 18 15:12:56 2017 -0700

--
 .../asyncevents/AsyncEventQueueType.java|  35 ++
 .../asyncevents/AsyncEventService.java  |   4 +-
 .../asyncevents/AsyncEventServiceImpl.java  | 463 +--
 .../index/IndexProcessorFig.java|  18 +
 .../corepersistence/index/ReIndexAction.java|   5 +-
 .../index/ReIndexServiceImpl.java   |   3 +-
 .../read/traverse/AbstractReadGraphFilter.java  |  11 +-
 .../AbstractReadReverseGraphFilter.java |  11 +-
 8 files changed, 292 insertions(+), 258 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/usergrid/blob/570e1ab4/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventQueueType.java
--
diff --git 
a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventQueueType.java
 
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventQueueType.java
new file mode 100644
index 000..4b91e17
--- /dev/null
+++ 
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventQueueType.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.asyncevents;
+
+public enum AsyncEventQueueType {
+REGULAR ("regular"), UTILITY("utility"), DELETE("delete");
+
+private String displayName;
+AsyncEventQueueType(String displayName) {
+this.displayName = displayName;
+}
+
+@Override
+public String toString() {
+return displayName;
+}
+}
+

http://git-wip-us.apache.org/repos/asf/usergrid/blob/570e1ab4/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventService.java
--
diff --git 
a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventService.java
 
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventService.java
index cab4e3e..9e346cf 100644
--- 
a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventService.java
+++ 
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventService.java
@@ -84,9 +84,9 @@ public interface AsyncEventService extends ReIndexAction {
 /**
  *
  * @param indexOperationMessage
- * @param forUtilityQueue
+ * @param queueType
  */
-void queueIndexOperationMessage(final IndexOperationMessage 
indexOperationMessage, boolean forUtilityQueue);
+void queueIndexOperationMessage(final IndexOperationMessage 
indexOperationMessage, AsyncEventQueueType queueType);
 
 /**
  * @param applicationScope

http://git-wip-us.apache.org/repos/asf/usergrid/blob/570e1ab4/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventServiceImpl.java
--
diff --git 
a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventServiceImpl.java
 
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventServiceImpl.java
index 9501ad3..0e55e9b 100644
--- 
a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventServiceImpl.java
+++ 
b/stack/core/src/main/java/org/apache/use

[05/27] usergrid git commit: fix html encode in jsp

[mrusso]

fix html encode in jsp


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/be2f69c3
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/be2f69c3
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/be2f69c3

Branch: refs/heads/expose-reindex
Commit: be2f69c37cab7214382d99e36e27347fc9f3b09b
Parents: 068263e
Author: Mike Dunker 
Authored: Fri Jul 28 14:11:31 2017 -0700
Committer: Mike Dunker 
Committed: Thu Aug 17 09:47:00 2017 -0700

--
 .../org/apache/usergrid/rest/TestResource/error.jsp  |  4 ++--
 .../org/apache/usergrid/rest/TestResource/test.jsp   |  5 +++--
 .../ApplicationResource/authorize_form.jsp   | 15 ---
 .../rest/applications/ApplicationResource/error.jsp  |  4 ++--
 .../applications/users/UserResource/activate.jsp |  4 ++--
 .../rest/applications/users/UserResource/confirm.jsp |  4 ++--
 .../rest/applications/users/UserResource/error.jsp   |  4 ++--
 .../users/UserResource/resetpw_email_form.jsp| 11 ++-
 .../users/UserResource/resetpw_email_success.jsp |  4 ++--
 .../users/UserResource/resetpw_set_form.jsp  |  9 +
 .../users/UserResource/resetpw_set_success.jsp   |  4 ++--
 .../rest/applications/users/UsersResource/error.jsp  |  4 ++--
 .../users/UsersResource/resetpw_email_form.jsp   |  9 +
 .../users/UsersResource/resetpw_email_success.jsp|  4 ++--
 .../management/ManagementResource/authorize_form.jsp | 15 ---
 .../rest/management/ManagementResource/error.jsp |  4 ++--
 .../organizations/OrganizationResource/activate.jsp  |  4 ++--
 .../organizations/OrganizationResource/confirm.jsp   |  4 ++--
 .../organizations/OrganizationResource/error.jsp |  4 ++--
 .../rest/management/users/UserResource/activate.jsp  |  4 ++--
 .../rest/management/users/UserResource/confirm.jsp   |  4 ++--
 .../rest/management/users/UserResource/error.jsp |  4 ++--
 .../users/UserResource/resetpw_email_form.jsp|  9 +
 .../users/UserResource/resetpw_email_success.jsp |  4 ++--
 .../users/UserResource/resetpw_set_form.jsp  |  8 
 .../users/UserResource/resetpw_set_success.jsp   |  4 ++--
 .../rest/management/users/UsersResource/error.jsp|  4 ++--
 .../users/UsersResource/resetpw_email_form.jsp   |  6 +++---
 .../users/UsersResource/resetpw_email_success.jsp|  4 ++--
 29 files changed, 87 insertions(+), 80 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/usergrid/blob/be2f69c3/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/TestResource/error.jsp
--
diff --git 
a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/TestResource/error.jsp
 
b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/TestResource/error.jsp
index be184b1..d02ad40 100644
--- 
a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/TestResource/error.jsp
+++ 
b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/TestResource/error.jsp
@@ -27,7 +27,7 @@ limitations under the License.
 
 
 
-   An error occurred .
+   An error occurred .
 
 
-
\ No newline at end of file
+

http://git-wip-us.apache.org/repos/asf/usergrid/blob/be2f69c3/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/TestResource/test.jsp
--
diff --git 
a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/TestResource/test.jsp
 
b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/TestResource/test.jsp
index 83a6ad1..68c12f2 100644
--- 
a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/TestResource/test.jsp
+++ 
b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/TestResource/test.jsp
@@ -1,5 +1,6 @@
 <%@ page language="java" contentType="text/html; charset=ISO-8859-1"
 pageEncoding="ISO-8859-1"%>
+<%@ taglib uri="http://java.sun.com/jsp/jstl/functions"; prefix="fn"%>
 

[27/27] usergrid git commit: Initial commit for exposing collection re-index to non sysadmin users.

[mrusso]

Initial commit for exposing collection re-index to non sysadmin users.


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/ec8e5c7c
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/ec8e5c7c
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/ec8e5c7c

Branch: refs/heads/expose-reindex
Commit: ec8e5c7ca42b52dc128a653bd7b6c46eebda11fd
Parents: 4a65910
Author: Michael Russo 
Authored: Wed Jun 28 11:07:15 2017 -0700
Committer: Michael Russo 
Committed: Wed Oct 4 19:02:59 2017 -0700

--
 .../corepersistence/index/ReIndexService.java   |  33 +-
 .../index/ReIndexServiceImpl.java   | 106 +--
 .../rest/applications/CollectionResource.java   |  27 -
 .../usergrid/rest/system/IndexResource.java |  78 --
 4 files changed, 196 insertions(+), 48 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/usergrid/blob/ec8e5c7c/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexService.java
--
diff --git 
a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexService.java
 
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexService.java
index b9238e5..d37f117 100644
--- 
a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexService.java
+++ 
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexService.java
@@ -20,6 +20,8 @@
 package org.apache.usergrid.corepersistence.index;
 
 
+import org.apache.usergrid.utils.StringUtils;
+
 /**
  * An interface for re-indexing all entities in an application
  */
@@ -47,6 +49,13 @@ public interface ReIndexService {
  */
 ReIndexStatus getStatus( final String jobId );
 
+/**
+ * Get the status of a collection job
+ * @param collectionName The collectionName for the rebuild index
+ * @return
+ */
+ReIndexStatus getStatusForCollection( final String appIdString, final 
String collectionName );
+
 
 /**
  * The response when requesting a re-index operation
@@ -56,14 +65,27 @@ public interface ReIndexService {
 final Status status;
 final long numberProcessed;
 final long lastUpdated;
+final String collectionName;
 
 
 public ReIndexStatus( final String jobId, final Status status, final 
long numberProcessed,
-  final long lastUpdated ) {
-this.jobId = jobId;
+  final long lastUpdated, final String 
collectionName ) {
+
+if(StringUtils.isNotEmpty(jobId)){
+this.jobId = jobId;
+}else {
+this.jobId = "";
+}
+
 this.status = status;
 this.numberProcessed = numberProcessed;
 this.lastUpdated = lastUpdated;
+
+if(StringUtils.isNotEmpty(collectionName)){
+this.collectionName = collectionName;
+}else {
+this.collectionName = "";
+}
 }
 
 
@@ -74,6 +96,13 @@ public interface ReIndexService {
 return jobId;
 }
 
+/**
+ * Get the jobId used to resume this operation
+ */
+public String getCollectionName() {
+return collectionName;
+}
+
 
 /**
  * Get the last updated time, as a long

http://git-wip-us.apache.org/repos/asf/usergrid/blob/ec8e5c7c/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexServiceImpl.java
--
diff --git 
a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexServiceImpl.java
 
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexServiceImpl.java
index 05602fc..d4fb249 100644
--- 
a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexServiceImpl.java
+++ 
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexServiceImpl.java
@@ -75,6 +75,7 @@ public class ReIndexServiceImpl implements ReIndexService {
 private static final String MAP_COUNT_KEY = "count";
 private static final String MAP_STATUS_KEY = "status";
 private static final String MAP_UPDATED_KEY = "lastUpdated";
+private static final String MAP_SEPARATOR = "|||";
 
 
 private final AllApplicationsObservable allApplicationsObservable;
@@ -140,7 +141,9 @@ public class ReIndexServiceImpl implements ReIndexService {
 
 // create an observable that loads a batch to be indexed
 
-if(reIndexRequestBuilder.getCollectionName().isPresent()) {
+final boolean isForCollection = 
reIndexRequestBuilder.getCollectionName().isPresent();
+
+if(isForCollec

[25/27] usergrid git commit: Allow submission to SNS/SQS via sync client

[mrusso]

Allow submission to SNS/SQS via sync client


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/9a151089
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/9a151089
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/9a151089

Branch: refs/heads/expose-reindex
Commit: 9a15108924b29997c9ca440b467ab8006a8f8acb
Parents: 173be8a
Author: Peter Johnson 
Authored: Wed Sep 27 08:19:20 2017 -0700
Committer: Peter Johnson 
Committed: Wed Sep 27 08:19:20 2017 -0700

--
 .../corepersistence/CpEntityManager.java|  75 +++--
 .../corepersistence/CpRelationManager.java  |  52 -
 .../asyncevents/AsyncEventService.java  |   6 +-
 .../asyncevents/AsyncEventServiceImpl.java  |  44 +---
 .../asyncevents/EventBuilderImpl.java   |   2 +
 .../corepersistence/util/CpCollectionUtils.java | 104 +
 .../index/AsyncIndexServiceTest.java|   2 +-
 .../persistence/queue/LegacyQueueFig.java   |   5 +
 .../persistence/queue/LegacyQueueManager.java   |  11 +-
 .../persistence/queue/LocalQueueManager.java|  11 +-
 .../queue/impl/QakkaQueueManager.java   |  12 +-
 .../queue/impl/SNSQueueManagerImpl.java | 112 +--
 .../queue/LegacyQueueManagerTest.java   |   2 +-
 .../impl/ApplicationQueueManagerImpl.java   |   2 +-
 .../services/queues/ImportQueueManager.java |   8 +-
 15 files changed, 341 insertions(+), 107 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/usergrid/blob/9a151089/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
--
diff --git 
a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
 
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
index 68c4ef0..1dc4a89 100644
--- 
a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
+++ 
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
@@ -34,6 +34,7 @@ import 
org.apache.usergrid.corepersistence.index.CollectionSettingsFactory;
 import org.apache.usergrid.corepersistence.index.CollectionSettingsScopeImpl;
 import org.apache.usergrid.corepersistence.service.CollectionService;
 import org.apache.usergrid.corepersistence.service.ConnectionService;
+import org.apache.usergrid.corepersistence.util.CpCollectionUtils;
 import org.apache.usergrid.corepersistence.util.CpEntityMapUtils;
 import org.apache.usergrid.corepersistence.util.CpNamingUtils;
 import org.apache.usergrid.mq.QueueManager;
@@ -161,7 +162,6 @@ public class CpEntityManager implements EntityManager {
 
 public QueueManagerFactory queueManagerFactory;
 
-
 ///** Short-term cache to keep us from reloading same Entity during 
single request. */
 //private LoadingCache entityCache;
 
@@ -521,6 +521,10 @@ public class CpEntityManager implements EntityManager {
 
 cpEntity = CpEntityMapUtils.fromMap( cpEntity, entity.getProperties(), 
entity.getType(), true );
 
+String entityType = cpEntity.getId().getType();
+boolean skipIndexingForType = skipIndexingForType(entityType);
+Boolean asyncIndex = asyncIndexingForType(entityType);
+
 try {
 
 String region = lookupAuthoritativeRegionForType( entity.getType() 
);
@@ -546,38 +550,32 @@ public class CpEntityManager implements EntityManager {
 handleWriteUniqueVerifyException( entity, wuve );
 }
 
-if ( !skipIndexingForType( cpEntity.getId().getType() ) ) {
+if (!skipIndexingForType) {
+indexEntity(cpEntity, asyncIndex);
+deIndexOldVersionsOfEntity(cpEntity);
+}
+}
 
-// queue an event to update the new entity
-indexService.queueEntityIndexUpdate( applicationScope, cpEntity, 0 
);
+private void 
indexEntity(org.apache.usergrid.persistence.model.entity.Entity cpEntity, 
Boolean async) {
+// queue an event to update the new entity
+indexService.queueEntityIndexUpdate( applicationScope, cpEntity, 0 , 
async);
+}
 
-// queue up an event to clean-up older versions than this one from 
the index
-if (entityManagerFig.getDeindexOnUpdate()) {
-indexService.queueDeIndexOldVersion( applicationScope, 
cpEntity.getId(), cpEntity.getVersion());
-}
+private void 
deIndexOldVersionsOfEntity(org.apache.usergrid.persistence.model.entity.Entity 
cpEntity) {
+// queue up an event to clean-up older versions than this one from the 
index
+if (entityManagerFig.getDeindexOnUpdate()) {
+indexService.queueDeIndexOldVersion( applicationScope, 
cpEntity.getId(), cpEnti

[17/27] usergrid git commit: Add ability to walk through a collection and delete all the entities, optionally up to a certain timestamp. Modeled after reindex services.

[mrusso]

Add ability to walk through a collection and delete all the entities, 
optionally up to a certain timestamp. Modeled after reindex services.


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/11823f29
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/11823f29
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/11823f29

Branch: refs/heads/expose-reindex
Commit: 11823f294dfae762754ef1c4da8a5ee573107968
Parents: 3f7afcd
Author: Mike Dunker 
Authored: Mon Aug 28 14:46:17 2017 -0700
Committer: Mike Dunker 
Committed: Mon Aug 28 14:46:17 2017 -0700

--
 .../usergrid/corepersistence/CoreModule.java|   1 +
 .../asyncevents/AsyncEventService.java  |   3 +-
 .../asyncevents/AsyncEventServiceImpl.java  |  67 +++--
 .../asyncevents/EventBuilder.java   |  13 +-
 .../asyncevents/EventBuilderImpl.java   |  45 ++-
 .../asyncevents/model/EntityDeleteEvent.java|  27 +-
 .../index/CollectionDeleteAction.java   |  43 +++
 .../index/CollectionDeleteRequestBuilder.java   |  92 ++
 .../CollectionDeleteRequestBuilderImpl.java | 146 +
 .../index/CollectionDeleteService.java  | 108 +++
 .../index/CollectionDeleteServiceImpl.java  | 299 +++
 .../index/IndexProcessorFig.java|   9 +
 .../index/ReIndexServiceImpl.java   |   2 +-
 .../persistence/CollectionDeleteTest.java   | 266 +
 .../resources/usergrid-custom-test.properties   |   2 +
 .../rest/applications/CollectionResource.java   | 130 +++-
 16 files changed, 1217 insertions(+), 36 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/usergrid/blob/11823f29/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
--
diff --git 
a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java 
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
index ec6b775..a0748e6 100644
--- 
a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
+++ 
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
@@ -140,6 +140,7 @@ public class CoreModule extends AbstractModule {
 
 
 bind( ReIndexService.class ).to( ReIndexServiceImpl.class );
+bind( CollectionDeleteService.class ).to( 
CollectionDeleteServiceImpl.class );
 
 bind( ExportService.class ).to( ExportServiceImpl.class );
 

http://git-wip-us.apache.org/repos/asf/usergrid/blob/11823f29/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventService.java
--
diff --git 
a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventService.java
 
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventService.java
index 9e346cf..04eaf4c 100644
--- 
a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventService.java
+++ 
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventService.java
@@ -20,6 +20,7 @@
 package org.apache.usergrid.corepersistence.asyncevents;
 
 
+import org.apache.usergrid.corepersistence.index.CollectionDeleteAction;
 import org.apache.usergrid.corepersistence.index.ReIndexAction;
 import org.apache.usergrid.persistence.core.scope.ApplicationScope;
 import org.apache.usergrid.persistence.graph.Edge;
@@ -33,7 +34,7 @@ import java.util.UUID;
 /**
  * Low level queue service for events in the entity.  These events are fire 
and forget, and will always be asynchronous
  */
-public interface AsyncEventService extends ReIndexAction {
+public interface AsyncEventService extends ReIndexAction, 
CollectionDeleteAction {
 
 
 /**

http://git-wip-us.apache.org/repos/asf/usergrid/blob/11823f29/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventServiceImpl.java
--
diff --git 
a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventServiceImpl.java
 
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventServiceImpl.java
index 428772f..3d06cae 100644
--- 
a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventServiceImpl.java
+++ 
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventServiceImpl.java
@@ -75,9 +75,6 @@ import java.util.concurrent.atomic.AtomicLong;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
-import static org.apache.commons.lang.StringUtils.indexOf;
-import static org.apache.commons.lang.StringUtils

[23/27] usergrid git commit: add missing fn reference in jsps

[mrusso]

add missing fn reference in jsps


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/7f40d685
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/7f40d685
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/7f40d685

Branch: refs/heads/expose-reindex
Commit: 7f40d685401397b44e2026ac6bd3af446c02ddbd
Parents: 9e16e2f
Author: Mike Dunker 
Authored: Thu Sep 21 14:29:05 2017 -0700
Committer: Mike Dunker 
Committed: Thu Sep 21 14:29:05 2017 -0700

--
 .../rest/management/users/UserResource/resetpw_set_form.jsp | 1 +
 .../rest/management/users/UsersResource/resetpw_email_form.jsp  | 1 +
 2 files changed, 2 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/usergrid/blob/7f40d685/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/users/UserResource/resetpw_set_form.jsp
--
diff --git 
a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/users/UserResource/resetpw_set_form.jsp
 
b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/users/UserResource/resetpw_set_form.jsp
index 6334466..c0203ce 100644
--- 
a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/users/UserResource/resetpw_set_form.jsp
+++ 
b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/users/UserResource/resetpw_set_form.jsp
@@ -4,6 +4,7 @@
 <%@ page import="net.tanesha.recaptcha.ReCaptchaFactory"%>
 <%@ page import="org.apache.usergrid.rest.AbstractContextResource"%>
 <%@ taglib uri="http://java.sun.com/jsp/jstl/core"; prefix="c"%>
+<%@ taglib uri="http://java.sun.com/jsp/jstl/functions"; prefix="fn"%>
 

[11/27] usergrid git commit: Basic code improvements 1. Increase some testing timeouts. 2. improve some logging statements. 3. Fix duplicate dependency in cored-persistence pom.xml 4. Accept SNS or DI

[mrusso]

Basic code improvements
1. Increase some testing timeouts.
2. improve some logging statements.
3. Fix duplicate dependency in cored-persistence pom.xml
4. Accept SNS or DISTRIBUTED_SNS as SNS queue types.
5. Drop missed test keyspaces for qakka tests.


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/400365c3
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/400365c3
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/400365c3

Branch: refs/heads/expose-reindex
Commit: 400365c36eb1a46ba76ca3b2cf8f2db6670227b1
Parents: 25f1b0e
Author: Mike Dunker 
Authored: Thu Aug 17 08:59:18 2017 -0700
Committer: Mike Dunker 
Committed: Thu Aug 17 13:37:06 2017 -0700

--
 .../main/resources/usergrid-default.properties|  2 +-
 .../resources/usergrid-custom-test.properties |  2 +-
 .../actorsystem/ActorSystemManagerImpl.java   |  3 ++-
 .../persistence/index/impl/EsEntityIndexImpl.java |  2 +-
 stack/corepersistence/queue/pom.xml   | 18 ++
 .../persistence/queue/guice/QueueModule.java  |  2 +-
 .../persistence/qakka/KeyspaceDropper.java|  6 +-
 .../qakka/core/QueueMessageManagerTest.java   |  2 +-
 .../resources/usergrid-custom-test.properties |  2 +-
 9 files changed, 19 insertions(+), 20 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/usergrid/blob/400365c3/stack/config/src/main/resources/usergrid-default.properties
--
diff --git a/stack/config/src/main/resources/usergrid-default.properties 
b/stack/config/src/main/resources/usergrid-default.properties
index 77b047e..bf1f5e7 100644
--- a/stack/config/src/main/resources/usergrid-default.properties
+++ b/stack/config/src/main/resources/usergrid-default.properties
@@ -112,7 +112,7 @@ cassandra.embedded=false
 
 # Read timeout for an individual request (in millseconds)
 #
-#cassandra.timeout=5000
+#cassandra.timeout=2
 
 
 # Set the credentials used for Cassandra, if any.

http://git-wip-us.apache.org/repos/asf/usergrid/blob/400365c3/stack/core/src/test/resources/usergrid-custom-test.properties
--
diff --git a/stack/core/src/test/resources/usergrid-custom-test.properties 
b/stack/core/src/test/resources/usergrid-custom-test.properties
index e32b146..df253f0 100644
--- a/stack/core/src/test/resources/usergrid-custom-test.properties
+++ b/stack/core/src/test/resources/usergrid-custom-test.properties
@@ -16,7 +16,7 @@
 # with ug.heapmax=5000m and ug.heapmin=3000m (set in Maven settings.xml)
 #cassandra.connections=30
 cassandra.timeout.pool=2
-cassandra.timeout=2
+cassandra.timeout=25000
 
 
 #Not a good number for real systems.  Write shards should be 2x cluster size 
from our tests

http://git-wip-us.apache.org/repos/asf/usergrid/blob/400365c3/stack/corepersistence/actorsystem/src/main/java/org/apache/usergrid/persistence/actorsystem/ActorSystemManagerImpl.java
--
diff --git 
a/stack/corepersistence/actorsystem/src/main/java/org/apache/usergrid/persistence/actorsystem/ActorSystemManagerImpl.java
 
b/stack/corepersistence/actorsystem/src/main/java/org/apache/usergrid/persistence/actorsystem/ActorSystemManagerImpl.java
index bfa3abe..ea9ada8 100644
--- 
a/stack/corepersistence/actorsystem/src/main/java/org/apache/usergrid/persistence/actorsystem/ActorSystemManagerImpl.java
+++ 
b/stack/corepersistence/actorsystem/src/main/java/org/apache/usergrid/persistence/actorsystem/ActorSystemManagerImpl.java
@@ -377,6 +377,7 @@ public class ActorSystemManagerImpl implements 
ActorSystemManager {
 Iterator messageTypes = 
routerProducer.getMessageTypes().iterator();
 while ( messageTypes.hasNext() ) {
 Class messageType = messageTypes.next();
+logger.info("createClusterSystem: routerProducer {}: 
message type={}", routerProducer.getRouterPath(), messageType.getName());
 routersByMessageType.put( messageType, 
routerProducer.getRouterPath() );
 }
 }
@@ -467,7 +468,7 @@ public class ActorSystemManagerImpl implements 
ActorSystemManager {
 if (started) {
 logger.info( "ClientActor [{}] has started", ra.path() );
 } else {
-throw new RuntimeException( "ClientActor ["+ra.path()+"] did not 
start in time" );
+throw new RuntimeException( "ClientActor ["+ra.path()+"] did not 
start in time, validate that akka seeds are configured properly" );
 }
 }
 

http://git-wip-us.apache.org/repos/asf/usergrid/blob/400365c3/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityI

[22/27] usergrid git commit: Bypass check of app APM setting if APM not enabled

[mrusso]

Bypass check of app APM setting if APM not enabled


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/9e16e2f0
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/9e16e2f0
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/9e16e2f0

Branch: refs/heads/expose-reindex
Commit: 9e16e2f006676d9e16997d378224e06a8683c744
Parents: 4170295
Author: Mike Dunker 
Authored: Thu Sep 14 15:07:30 2017 -0700
Committer: Mike Dunker 
Committed: Thu Sep 14 15:07:30 2017 -0700

--
 .../apache/usergrid/rest/applications/ApplicationResource.java  | 4 
 .../java/org/apache/usergrid/management/ManagementService.java  | 2 ++
 .../usergrid/management/cassandra/ManagementServiceImpl.java| 5 +
 3 files changed, 11 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/usergrid/blob/9e16e2f0/stack/rest/src/main/java/org/apache/usergrid/rest/applications/ApplicationResource.java
--
diff --git 
a/stack/rest/src/main/java/org/apache/usergrid/rest/applications/ApplicationResource.java
 
b/stack/rest/src/main/java/org/apache/usergrid/rest/applications/ApplicationResource.java
index 7479a90..a6eefb1 100644
--- 
a/stack/rest/src/main/java/org/apache/usergrid/rest/applications/ApplicationResource.java
+++ 
b/stack/rest/src/main/java/org/apache/usergrid/rest/applications/ApplicationResource.java
@@ -621,6 +621,10 @@ public class ApplicationResource extends 
CollectionResource {
 public Object getAPMConfig( @Context UriInfo ui,
  @QueryParam("callback") 
@DefaultValue("callback") String callback )
 throws Exception {
+if (!management.isAPMEnabled()) {
+// no need to check app for setting
+throw new EntityNotFoundException("apigeeMobileConfig not 
enabled.");
+}
 EntityManager em = emf.getEntityManager( applicationId );
 Object value = em.getProperty( new SimpleEntityRef( 
Application.ENTITY_TYPE, applicationId ),
 APIGEE_MOBILE_APM_CONFIG_JSON_KEY );

http://git-wip-us.apache.org/repos/asf/usergrid/blob/9e16e2f0/stack/services/src/main/java/org/apache/usergrid/management/ManagementService.java
--
diff --git 
a/stack/services/src/main/java/org/apache/usergrid/management/ManagementService.java
 
b/stack/services/src/main/java/org/apache/usergrid/management/ManagementService.java
index 8b840d6..3d0c9fb 100644
--- 
a/stack/services/src/main/java/org/apache/usergrid/management/ManagementService.java
+++ 
b/stack/services/src/main/java/org/apache/usergrid/management/ManagementService.java
@@ -343,6 +343,8 @@ public interface ManagementService {
 
Object registerAppWithAPM( OrganizationInfo orgInfo, ApplicationInfo 
appInfo ) throws Exception;
 
+   boolean isAPMEnabled() throws Exception;
+
/** For testing purposes only */
Properties getProperties();
 

http://git-wip-us.apache.org/repos/asf/usergrid/blob/9e16e2f0/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java
--
diff --git 
a/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java
 
b/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java
index 89375fd..2302d8d 100644
--- 
a/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java
+++ 
b/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java
@@ -3404,6 +3404,11 @@ public class ManagementServiceImpl implements 
ManagementService {
 }
 
 @Override
+public boolean isAPMEnabled() throws Exception {
+return false;
+}
+
+@Override
 public Observable deleteAllEntities(final UUID applicationId,final int 
limit){
 if(applicationId.equals(CpNamingUtils.MANAGEMENT_APPLICATION_ID)){
 throw new IllegalArgumentException("Can't delete from management 
app");

[01/27] usergrid git commit: increase test cassandra.timeout settings [Forced Update!]

[mrusso]

Repository: usergrid
Updated Branches:
  refs/heads/expose-reindex 148337ec8 -> ec8e5c7ca (forced update)


increase test cassandra.timeout settings


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/f22c6c4b
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/f22c6c4b
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/f22c6c4b

Branch: refs/heads/expose-reindex
Commit: f22c6c4bbe89b6fa94ce223b3381e23ccda4f71c
Parents: a6fee78
Author: Mike Dunker 
Authored: Wed Aug 16 20:07:45 2017 -0700
Committer: Mike Dunker 
Committed: Wed Aug 16 20:07:45 2017 -0700

--
 .../common/src/test/resources/usergrid-UNIT.properties | 2 +-
 stack/rest/src/test/resources/usergrid-custom-test.properties  | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/usergrid/blob/f22c6c4b/stack/corepersistence/common/src/test/resources/usergrid-UNIT.properties
--
diff --git 
a/stack/corepersistence/common/src/test/resources/usergrid-UNIT.properties 
b/stack/corepersistence/common/src/test/resources/usergrid-UNIT.properties
index 0a43209..13b2937 100644
--- a/stack/corepersistence/common/src/test/resources/usergrid-UNIT.properties
+++ b/stack/corepersistence/common/src/test/resources/usergrid-UNIT.properties
@@ -24,7 +24,7 @@ cassandra.port=9160
 cassandra.hosts=localhost
 cassandra.cluster_name=Usergrid
 collections.keyspace=Usergrid_Collections
-cassandra.timeout=1
+cassandra.timeout=2
 cassandra.embedded=true
 
 collections.keyspace.strategy.options=replication_factor:1

http://git-wip-us.apache.org/repos/asf/usergrid/blob/f22c6c4b/stack/rest/src/test/resources/usergrid-custom-test.properties
--
diff --git a/stack/rest/src/test/resources/usergrid-custom-test.properties 
b/stack/rest/src/test/resources/usergrid-custom-test.properties
index 615bedd..cbec81f 100644
--- a/stack/rest/src/test/resources/usergrid-custom-test.properties
+++ b/stack/rest/src/test/resources/usergrid-custom-test.properties
@@ -18,6 +18,7 @@
 cassandra.startup=external
 cassandra.connections=30
 cassandra.timeout.pool=2
+cassandra.timeout=2
 
 
 hystrix.threadpool.graph_user.coreSize=1200

[19/27] usergrid git commit: Give entity creation time to finish during collection delete test.

[mrusso]

Give entity creation time to finish during collection delete test.


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/3d3ba490
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/3d3ba490
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/3d3ba490

Branch: refs/heads/expose-reindex
Commit: 3d3ba49065018a74a51709ed23bb777826b21f78
Parents: 8e9eff8
Author: Mike Dunker 
Authored: Mon Aug 28 19:29:06 2017 -0700
Committer: Mike Dunker 
Committed: Mon Aug 28 19:29:06 2017 -0700

--
 .../org/apache/usergrid/persistence/CollectionDeleteTest.java | 3 +++
 1 file changed, 3 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/usergrid/blob/3d3ba490/stack/core/src/test/java/org/apache/usergrid/persistence/CollectionDeleteTest.java
--
diff --git 
a/stack/core/src/test/java/org/apache/usergrid/persistence/CollectionDeleteTest.java
 
b/stack/core/src/test/java/org/apache/usergrid/persistence/CollectionDeleteTest.java
index ddf2c68..d062ef4 100644
--- 
a/stack/core/src/test/java/org/apache/usergrid/persistence/CollectionDeleteTest.java
+++ 
b/stack/core/src/test/java/org/apache/usergrid/persistence/CollectionDeleteTest.java
@@ -111,6 +111,9 @@ public class CollectionDeleteTest extends AbstractCoreIT {
 }
 
 logger.info("Created {} entities", ENTITIES_TO_DELETE);
+
+app.waitForQueueDrainAndRefreshIndex(1);
+
 long timeFirstPutDone = System.currentTimeMillis();
 logger.info("timeFirstPutDone={}", timeFirstPutDone);
 

[06/27] usergrid git commit: fix broken recaptcha

[mrusso]

fix broken recaptcha


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/459163b8
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/459163b8
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/459163b8

Branch: refs/heads/expose-reindex
Commit: 459163b8ddb4b80ebb43273a8646ee9c8a6fe2bd
Parents: be2f69c
Author: Mike Dunker 
Authored: Wed Aug 9 18:39:23 2017 -0700
Committer: Mike Dunker 
Committed: Thu Aug 17 09:47:40 2017 -0700

--
 .../rest/applications/ApplicationResource/authorize_form.jsp | 2 +-
 .../rest/applications/users/UserResource/resetpw_email_form.jsp  | 4 ++--
 .../rest/applications/users/UserResource/resetpw_set_form.jsp| 2 +-
 .../rest/applications/users/UsersResource/resetpw_email_form.jsp | 4 ++--
 .../rest/management/ManagementResource/authorize_form.jsp| 2 +-
 .../rest/management/users/UserResource/resetpw_email_form.jsp| 4 ++--
 .../rest/management/users/UserResource/resetpw_set_form.jsp  | 2 +-
 .../rest/management/users/UsersResource/resetpw_email_form.jsp   | 4 ++--
 8 files changed, 12 insertions(+), 12 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/usergrid/blob/459163b8/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/ApplicationResource/authorize_form.jsp
--
diff --git 
a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/ApplicationResource/authorize_form.jsp
 
b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/ApplicationResource/authorize_form.jsp
index 0079bcf..ed934a7 100644
--- 
a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/ApplicationResource/authorize_form.jsp
+++ 
b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/ApplicationResource/authorize_form.jsp
@@ -29,7 +29,7 @@ limitations under the License.
 
 

-   ${fn:escapeXml(it.errorMsg)}
+   ${fn:escapeXml(it.errorMsg)}




http://git-wip-us.apache.org/repos/asf/usergrid/blob/459163b8/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/users/UserResource/resetpw_email_form.jsp
--
diff --git 
a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/users/UserResource/resetpw_email_form.jsp
 
b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/users/UserResource/resetpw_email_form.jsp
index 59026bf..5230ea7 100644
--- 
a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/users/UserResource/resetpw_email_form.jsp
+++ 
b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/users/UserResource/resetpw_email_form.jsp
@@ -27,7 +27,7 @@ limitations under the License.
 
 

-   
+   
${fn:escapeXml(it.errorMsg)}


@@ -37,7 +37,7 @@ limitations under the License.



-   ${fn:escapeXml(it.reCaptchaHtml)}
+   ${it.reCaptchaHtml}




http://git-wip-us.apache.org/repos/asf/usergrid/blob/459163b8/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/users/UserResource/resetpw_set_form.jsp
--
diff --git 
a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/users/UserResource/resetpw_set_form.jsp
 
b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/users/UserResource/resetpw_set_form.jsp
index 19b6528..c0203ce 100644
--- 
a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/users/UserResource/resetpw_set_form.jsp
+++ 
b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/users/UserResource/resetpw_set_form.jsp
@@ -31,7 +31,7 @@ limitations under the License.
 
 

-   ${fn:escapeXml(it.errorMsg)}
+   ${fn:escapeXml(it.errorMsg)}




http://git-wip-us.apache.org/repos/asf/usergrid/blob/459163b8/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/users/UsersResource/resetpw_email_form.jsp
--
diff --git 
a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applicat

[10/27] usergrid git commit: add a little more wait time

[mrusso]

add a little more wait time


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/25f1b0e5
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/25f1b0e5
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/25f1b0e5

Branch: refs/heads/expose-reindex
Commit: 25f1b0e5a86f51d0da465d78cb0c5fbd1a006a7d
Parents: d06dbc9
Author: Mike Dunker 
Authored: Wed Aug 16 22:30:34 2017 -0700
Committer: Mike Dunker 
Committed: Thu Aug 17 11:59:53 2017 -0700

--
 .../apache/usergrid/rest/applications/queries/QueryTestBase.java   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/usergrid/blob/25f1b0e5/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/QueryTestBase.java
--
diff --git 
a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/QueryTestBase.java
 
b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/QueryTestBase.java
index 0adafef..1eec9ae 100644
--- 
a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/QueryTestBase.java
+++ 
b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/QueryTestBase.java
@@ -65,7 +65,7 @@ public class QueryTestBase  extends AbstractRestIT {
 logger.info(entities[i].entrySet().toString());
 }
 //refresh the index so that they are immediately searchable
-this.waitForQueueDrainAndRefreshIndex();
+this.waitForQueueDrainAndRefreshIndex(1000);
 
 return entities;
 }

[07/27] usergrid git commit: add password complexity check before submitting during reset password flow

[mrusso]

add password complexity check before submitting during reset password flow


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/70de6fde
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/70de6fde
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/70de6fde

Branch: refs/heads/expose-reindex
Commit: 70de6fde4438bd4ba750806e5b4a23cd9aa07d9b
Parents: 459163b
Author: Mike Dunker 
Authored: Thu Aug 17 11:02:36 2017 -0700
Committer: Mike Dunker 
Committed: Thu Aug 17 11:02:36 2017 -0700

--
 .../usergrid/rest/applications/users/UserResource.java|  9 +
 .../usergrid/rest/management/users/UserResource.java  |  9 +
 .../org/apache/usergrid/management/ManagementService.java | 10 +-
 .../management/cassandra/ManagementServiceImpl.java   | 10 ++
 4 files changed, 33 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/usergrid/blob/70de6fde/stack/rest/src/main/java/org/apache/usergrid/rest/applications/users/UserResource.java
--
diff --git 
a/stack/rest/src/main/java/org/apache/usergrid/rest/applications/users/UserResource.java
 
b/stack/rest/src/main/java/org/apache/usergrid/rest/applications/users/UserResource.java
index 5435f7e..3e4542d 100644
--- 
a/stack/rest/src/main/java/org/apache/usergrid/rest/applications/users/UserResource.java
+++ 
b/stack/rest/src/main/java/org/apache/usergrid/rest/applications/users/UserResource.java
@@ -17,6 +17,7 @@
 package org.apache.usergrid.rest.applications.users;
 
 
+import java.util.Collection;
 import java.util.Map;
 import java.util.UUID;
 
@@ -465,6 +466,14 @@ public class UserResource extends ServiceResource {
 if ( ( password1 != null ) || ( password2 != null ) ) {
 if ( management.checkPasswordResetTokenForAppUser( 
getApplicationId(), getUserUuid(), token ) ) {
 if ( ( password1 != null ) && password1.equals( password2 
) ) {
+// validate password
+Collection violations = 
management.passwordPolicyCheck(password1, false);
+if (violations.size() > 0) {
+// password not valid
+errorMsg = 
management.getPasswordDescription(false);
+return handleViewable("resetpw_set_form", this, 
getOrganizationName());
+}
+
 management.setAppUserPassword( getApplicationId(), 
getUser().getUuid(), password1 );
 management.revokeAccessTokenForAppUser( token );
 return handleViewable( "resetpw_set_success", this, 
getOrganizationName() );

http://git-wip-us.apache.org/repos/asf/usergrid/blob/70de6fde/stack/rest/src/main/java/org/apache/usergrid/rest/management/users/UserResource.java
--
diff --git 
a/stack/rest/src/main/java/org/apache/usergrid/rest/management/users/UserResource.java
 
b/stack/rest/src/main/java/org/apache/usergrid/rest/management/users/UserResource.java
index b747aa4..1f80bc1 100644
--- 
a/stack/rest/src/main/java/org/apache/usergrid/rest/management/users/UserResource.java
+++ 
b/stack/rest/src/main/java/org/apache/usergrid/rest/management/users/UserResource.java
@@ -43,6 +43,7 @@ import javax.ws.rs.*;
 import javax.ws.rs.core.Context;
 import javax.ws.rs.core.MediaType;
 import javax.ws.rs.core.UriInfo;
+import java.util.Collection;
 import java.util.Map;
 import java.util.UUID;
 
@@ -297,6 +298,14 @@ public class UserResource extends AbstractContextResource {
 if ( ( password1 != null ) || ( password2 != null ) ) {
 if ( management.checkPasswordResetTokenForAdminUser( 
user.getUuid(), tokenInfo ) ) {
 if ( ( password1 != null ) && password1.equals( password2 
) ) {
+// validate password
+Collection violations = 
management.passwordPolicyCheck(password1, true);
+if (violations.size() > 0) {
+// password not valid
+errorMsg = management.getPasswordDescription(true);
+return handleViewable( "resetpw_set_form", this, 
organizationId );
+}
+
 management.setAdminUserPassword( user.getUuid(), 
password1 );
 management.revokeAccessTokenForAdminUser( 
user.getUuid(), token );
 loginEndpoint = 
properties.getProperty("usergrid.viewable.loginEndpoint");

http://git-wip-us.apache.org/repos/asf/usergrid/blob/70de6fde/stack/services/src/main/java/org/apache/usergrid/management/Ma

[16/27] usergrid git commit: 1. Add missing start for delete queue workers. 2. Add dropping of application local keyspace for testing. 3. Testing changes.

[mrusso]

1. Add missing start for delete queue workers.
2. Add dropping of application local keyspace for testing.
3. Testing changes.


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/3f7afcd8
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/3f7afcd8
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/3f7afcd8

Branch: refs/heads/expose-reindex
Commit: 3f7afcd8836ed9b589b69523d8bcc507a739d912
Parents: c201f1f
Author: Mike Dunker 
Authored: Mon Aug 21 17:01:48 2017 -0700
Committer: Mike Dunker 
Committed: Mon Aug 21 17:01:48 2017 -0700

--
 .../corepersistence/CpEntityManager.java|  3 +--
 .../asyncevents/AsyncEventServiceImpl.java  |  4 +++
 .../index/CollectionSettingsImpl.java   |  1 -
 .../usergrid/persistence/EntityManager.java |  2 +-
 .../persistence/entities/Application.java   | 26 
 .../usergrid/persistence/CoreSchemaManager.java | 16 
 .../usergrid/persistence/RebuildIndexTest.java  | 24 +-
 .../resources/usergrid-custom-test.properties   |  2 +-
 .../resources/usergrid-custom-test.properties   |  2 +-
 .../AbstractServiceNotificationIT.java  |  4 +--
 .../gcm/NotificationsServiceIT.java |  2 +-
 .../resources/usergrid-custom-test.properties   |  2 +-
 12 files changed, 67 insertions(+), 21 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/usergrid/blob/3f7afcd8/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
--
diff --git 
a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
 
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
index cdb4fc7..68c4ef0 100644
--- 
a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
+++ 
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
@@ -2471,10 +2471,9 @@ public class CpEntityManager implements EntityManager {
 
 final Entity entity;
 
-//this is the fall back, why isn't this writt
 if ( entityType == null ) {
  return null;
-//throw new EntityNotFoundException( String.format( "Counld not 
find type for uuid {}", uuid ) );
+//throw new EntityNotFoundException( String.format( "Could not 
find type for uuid {}", uuid ) );
 }
 
 entity = get( new SimpleEntityRef( entityType, uuid ) );

http://git-wip-us.apache.org/repos/asf/usergrid/blob/3f7afcd8/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventServiceImpl.java
--
diff --git 
a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventServiceImpl.java
 
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventServiceImpl.java
index 79a80c0..428772f 100644
--- 
a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventServiceImpl.java
+++ 
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventServiceImpl.java
@@ -875,6 +875,10 @@ public class AsyncEventServiceImpl implements 
AsyncEventService {
 startWorker(AsyncEventQueueType.UTILITY);
 }
 
+for (int i = 0; i < deleteCount; i++) {
+startWorker(AsyncEventQueueType.DELETE);
+}
+
 if( indexQueue instanceof SNSQueueManagerImpl) {
 logger.info("Queue manager implementation supports dead letters, 
start dead letter queue workers.");
 for (int i = 0; i < indexDeadCount; i++) {

http://git-wip-us.apache.org/repos/asf/usergrid/blob/3f7afcd8/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/CollectionSettingsImpl.java
--
diff --git 
a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/CollectionSettingsImpl.java
 
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/CollectionSettingsImpl.java
index 921777a..76f0b4b 100644
--- 
a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/CollectionSettingsImpl.java
+++ 
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/CollectionSettingsImpl.java
@@ -20,7 +20,6 @@ package org.apache.usergrid.corepersistence.index;
 
 import com.google.common.base.Optional;
 import com.google.inject.Inject;
-import com.google.inject.Singleton;
 import org.apache.usergrid.persistence.map.MapManager;
 import org.apache.usergrid.utils.JsonUtils;
 import org.slf4j.Logger;

http://git-wip-us.apache.org/repos/asf/usergrid/blob/3f7afcd8/stack/core/src/main/java/org/apache/usergrid/persistence/EntityM

[24/27] usergrid git commit: add reset_url as field in admin confirmation email

[mrusso]

add reset_url as field in admin confirmation email


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/173be8a4
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/173be8a4
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/173be8a4

Branch: refs/heads/expose-reindex
Commit: 173be8a4b968eda617f73a3baf05fa7870594657
Parents: 7f40d68
Author: Mike Dunker 
Authored: Fri Sep 22 14:51:55 2017 -0700
Committer: Mike Dunker 
Committed: Tue Sep 26 08:09:04 2017 -0700

--
 .../management/cassandra/ManagementServiceImpl.java| 13 ++---
 1 file changed, 10 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/usergrid/blob/173be8a4/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java
--
diff --git 
a/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java
 
b/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java
index 2302d8d..ab93563 100644
--- 
a/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java
+++ 
b/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java
@@ -2680,10 +2680,17 @@ public class ManagementServiceImpl implements 
ManagementService {
 String token = getConfirmationTokenForAdminUser(user.getUuid(), 0, 
organizationId);
 OrganizationConfig orgConfig = organizationId != null ?
 getOrganizationConfigByUuid(organizationId) : 
getOrganizationConfigForUserInfo(user);
-String confirmation_url = 
orgConfig.getFullUrl(WorkflowUrl.ADMIN_CONFIRMATION_URL,
-user.getUuid().toString()) + "?token=" + token;
+String confirmation_url = 
orgConfig.getFullUrl(WorkflowUrl.ADMIN_CONFIRMATION_URL, 
user.getUuid().toString()) +
+"?token=" + token;
+
+String reset_token = getPasswordResetTokenForAdminUser( 
user.getUuid(), 0, organizationId );
+String resetPropertyUrl = 
orgConfig.getFullUrlTemplate(WorkflowUrl.ADMIN_RESETPW_URL);
+String reset_url = String.format(resetPropertyUrl, 
user.getUuid().toString())
++ "?token=" + reset_token;
+
 sendAdminUserEmail(user, "User Account Confirmation: " + 
user.getEmail(),
-emailMsg(hashMap("confirm_email", 
user.getEmail()).map("confirmation_url", confirmation_url),
+emailMsg(hashMap("confirm_email", 
user.getEmail()).map("confirmation_url", confirmation_url)
+.map("reset_url", reset_url),
 PROPERTIES_EMAIL_ADMIN_CONFIRMATION));
 }
 

[18/27] usergrid git commit: Merge branch 'collectionClearJob'

[mrusso]

Merge branch 'collectionClearJob'


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/8e9eff8d
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/8e9eff8d
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/8e9eff8d

Branch: refs/heads/expose-reindex
Commit: 8e9eff8d9340fc85b7d4b544fa542e388974525a
Parents: 36e28a8 11823f2
Author: Mike Dunker 
Authored: Mon Aug 28 16:23:01 2017 -0700
Committer: Mike Dunker 
Committed: Mon Aug 28 16:23:01 2017 -0700

--
 .../main/resources/usergrid-default.properties  |   3 +-
 .../usergrid/corepersistence/CoreModule.java|   1 +
 .../corepersistence/CpEntityManager.java|   3 +-
 .../corepersistence/CpEntityManagerFactory.java |   2 +-
 .../corepersistence/EntityManagerFig.java   |   2 +-
 .../asyncevents/AsyncEventQueueType.java|  35 ++
 .../asyncevents/AsyncEventService.java  |   7 +-
 .../asyncevents/AsyncEventServiceImpl.java  | 545 +--
 .../asyncevents/EventBuilder.java   |  26 +-
 .../asyncevents/EventBuilderImpl.java   | 205 +--
 .../asyncevents/model/EntityDeleteEvent.java|  27 +-
 .../index/CollectionDeleteAction.java   |  43 ++
 .../index/CollectionDeleteRequestBuilder.java   |  92 
 .../CollectionDeleteRequestBuilderImpl.java | 146 +
 .../index/CollectionDeleteService.java  | 108 
 .../index/CollectionDeleteServiceImpl.java  | 299 ++
 .../index/CollectionSettingsImpl.java   |   1 -
 .../index/IndexProcessorFig.java|  27 +
 .../corepersistence/index/IndexService.java |  16 +-
 .../corepersistence/index/IndexServiceImpl.java |  56 +-
 .../corepersistence/index/ReIndexAction.java|   5 +-
 .../index/ReIndexServiceImpl.java   |   5 +-
 .../read/traverse/AbstractReadGraphFilter.java  |  42 +-
 .../AbstractReadReverseGraphFilter.java |  41 +-
 .../usergrid/persistence/EntityManager.java |   2 +-
 .../org/apache/usergrid/persistence/Query.java  |  17 +
 .../persistence/entities/Application.java   |  26 +
 .../rx/EdgesFromSourceObservableIT.java |   2 +-
 .../rx/EdgesToTargetObservableIT.java   |   4 +-
 .../persistence/CollectionDeleteTest.java   | 266 +
 .../usergrid/persistence/CoreSchemaManager.java |  16 +-
 .../usergrid/persistence/RebuildIndexTest.java  |  36 +-
 .../core/src/test/resources/project.properties  |   2 +-
 .../resources/usergrid-custom-test.properties   |   3 +-
 .../actorsystem/ActorSystemManagerImpl.java |   3 +-
 .../src/test/resources/dynamic-test.properties  |   1 +
 .../src/test/resources/usergrid-CHOP.properties |   1 +
 .../src/test/resources/usergrid-UNIT.properties |   1 +
 .../src/test/resources/usergrid.properties  |   1 +
 .../persistence/graph/GraphManager.java |   2 +-
 .../graph/impl/GraphManagerImpl.java|  21 +-
 .../graph/impl/SimpleSearchByEdgeType.java  |   9 +
 .../graph/impl/stage/NodeDeleteListener.java|   6 +-
 .../impl/stage/NodeDeleteListenerImpl.java  |  64 +--
 .../graph/serialization/EdgesObservable.java|   6 +-
 .../impl/EdgeMetadataSerializationV2Impl.java   |   1 -
 .../serialization/impl/EdgesObservableImpl.java |  22 +-
 .../impl/TargetIdObservableImpl.java|   4 +-
 .../impl/migration/EdgeDataMigrationImpl.java   |   2 +-
 .../graph/impl/NodeDeleteListenerTest.java  |   8 +-
 .../src/test/resources/usergrid-AWS.properties  |   1 +
 .../src/test/resources/usergrid-CHOP.properties |   1 +
 .../src/test/resources/usergrid-UNIT.properties |   1 +
 .../src/test/resources/usergrid.properties  |   1 +
 .../usergrid/persistence/index/EntityIndex.java |  12 +-
 .../index/impl/EsEntityIndexImpl.java   |  65 +--
 .../persistence/index/query/Identifier.java |   3 +-
 .../src/test/resources/dynamic-test.properties  |   1 +
 .../src/test/resources/usergrid-CHOP.properties |   1 +
 .../src/test/resources/usergrid-UNIT.properties |   1 +
 stack/corepersistence/queue/pom.xml |  18 +-
 .../queues/impl/QueueSerializationImpl.java |   2 +-
 .../persistence/queue/guice/QueueModule.java|   2 +-
 .../persistence/qakka/KeyspaceDropper.java  |   6 +-
 .../qakka/core/QueueMessageManagerTest.java |   2 +-
 .../queue/src/test/resources/qakka.properties   |   1 +
 stack/pom.xml   |   8 +-
 .../resources/usergrid-custom-test.properties   |   2 +
 stack/rest/pom.xml  |   4 +-
 .../rest/applications/CollectionResource.java   | 130 -
 .../rest/applications/users/UserResource.java   |   9 +
 .../rest/management/users/UserResource.java |  16 +
 .../apache/usergrid/rest/TestResource/error.jsp |   4 +-
 .../apache/usergrid/rest/TestResource/test.jsp  |   5 +-
 .../ApplicationResource/authorize_form.jsp  |  15 +-
 .../ap

[04/27] usergrid git commit: Add new parameter that can be used in the JSP viewables.

[mrusso]

Add new parameter that can be used in the JSP viewables.


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/068263ed
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/068263ed
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/068263ed

Branch: refs/heads/expose-reindex
Commit: 068263ed437883edeba1d23a024f62bb6e7b0700
Parents: a6fee78
Author: Michael Russo 
Authored: Wed May 24 21:45:04 2017 -0700
Committer: Mike Dunker 
Committed: Thu Aug 17 09:41:40 2017 -0700

--
 stack/config/src/main/resources/usergrid-default.properties   | 1 +
 .../apache/usergrid/rest/management/users/UserResource.java   | 7 +++
 2 files changed, 8 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/usergrid/blob/068263ed/stack/config/src/main/resources/usergrid-default.properties
--
diff --git a/stack/config/src/main/resources/usergrid-default.properties 
b/stack/config/src/main/resources/usergrid-default.properties
index d505a96..77b047e 100644
--- a/stack/config/src/main/resources/usergrid-default.properties
+++ b/stack/config/src/main/resources/usergrid-default.properties
@@ -786,6 +786,7 @@ usergrid.api.url.base=http://localhost:8080/ROOT
 # instead, use character class ([.] instead of backslash-period)
 usergrid.org.config.property.regex=usergrid[.]view[.].*
 
+usergrid.viewable.loginEndpoint=http://localhost:8080
 
 
 ###  Usergrid Email Templates  

http://git-wip-us.apache.org/repos/asf/usergrid/blob/068263ed/stack/rest/src/main/java/org/apache/usergrid/rest/management/users/UserResource.java
--
diff --git 
a/stack/rest/src/main/java/org/apache/usergrid/rest/management/users/UserResource.java
 
b/stack/rest/src/main/java/org/apache/usergrid/rest/management/users/UserResource.java
index 4cbe9b2..b747aa4 100644
--- 
a/stack/rest/src/main/java/org/apache/usergrid/rest/management/users/UserResource.java
+++ 
b/stack/rest/src/main/java/org/apache/usergrid/rest/management/users/UserResource.java
@@ -66,6 +66,9 @@ public class UserResource extends AbstractContextResource {
 
 String token = null;
 
+String loginEndpoint;
+
+
 
 public UserResource() {
 }
@@ -296,6 +299,7 @@ public class UserResource extends AbstractContextResource {
 if ( ( password1 != null ) && password1.equals( password2 
) ) {
 management.setAdminUserPassword( user.getUuid(), 
password1 );
 management.revokeAccessTokenForAdminUser( 
user.getUuid(), token );
+loginEndpoint = 
properties.getProperty("usergrid.viewable.loginEndpoint");
 return handleViewable( "resetpw_set_success", this, 
organizationId );
 }
 else {
@@ -342,6 +346,9 @@ public class UserResource extends AbstractContextResource {
 return errorMsg;
 }
 
+public String getLoginEndpoint() {
+return loginEndpoint;
+}
 
 public String getToken() {
 return token;

[21/27] usergrid git commit: Handle IndexMissingException when fetching the index size for an application.

[mrusso]

Handle IndexMissingException when fetching the index size for an application.


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/41702956
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/41702956
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/41702956

Branch: refs/heads/expose-reindex
Commit: 41702956d5aedac5a8e8d44f8f3df7af5ec365b9
Parents: c860521
Author: Michael Russo 
Authored: Tue Sep 12 18:42:43 2017 -0700
Committer: Michael Russo 
Committed: Tue Sep 12 18:42:43 2017 -0700

--
 .../index/impl/EsEntityIndexImpl.java   | 29 ++--
 1 file changed, 20 insertions(+), 9 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/usergrid/blob/41702956/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
--
diff --git 
a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
 
b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
index f4fae2b..3d2f576 100644
--- 
a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
+++ 
b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
@@ -70,6 +70,7 @@ import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.index.query.*;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.indices.IndexAlreadyExistsException;
+import org.elasticsearch.indices.IndexMissingException;
 import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.search.SearchHits;
 import org.elasticsearch.search.aggregations.metrics.sum.Sum;
@@ -804,15 +805,25 @@ public class EsEntityIndexImpl implements 
EntityIndex,VersionedData {
 }
 
 private long getIndexSize(){
-final IndicesStatsResponse statsResponse = esProvider.getClient()
-.admin()
-.indices()
-.prepareStats(indexLocationStrategy.getIndexInitialName())
-.all()
-.execute()
-.actionGet();
-final CommonStats indexStats = 
statsResponse.getIndex(indexLocationStrategy.getIndexInitialName()).getTotal();
-return indexStats.getStore().getSizeInBytes();
+long indexSize = 0L;
+final String indexName = indexLocationStrategy.getIndexInitialName();
+try {
+final IndicesStatsResponse statsResponse = esProvider.getClient()
+.admin()
+.indices()
+.prepareStats(indexName)
+.all()
+.execute()
+.actionGet();
+final CommonStats indexStats = 
statsResponse.getIndex(indexName).getTotal();
+indexSize = indexStats.getStore().getSizeInBytes();
+} catch (IndexMissingException e) {
+// if for some reason the index size does not exist,
+// log an error and we can assume size is 0 as it doesn't exist
+logger.error("Unable to get size for index {} due to 
IndexMissingException for app {}",
+indexName, 
indexLocationStrategy.getApplicationScope().getApplication().getUuid());
+}
+return indexSize;
 }
 
 @Override

usergrid git commit: Handle IndexMissingException when fetching the index size for an application.

[mrusso]

Repository: usergrid
Updated Branches:
  refs/heads/master c86052110 -> 41702956d


Handle IndexMissingException when fetching the index size for an application.


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/41702956
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/41702956
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/41702956

Branch: refs/heads/master
Commit: 41702956d5aedac5a8e8d44f8f3df7af5ec365b9
Parents: c860521
Author: Michael Russo 
Authored: Tue Sep 12 18:42:43 2017 -0700
Committer: Michael Russo 
Committed: Tue Sep 12 18:42:43 2017 -0700

--
 .../index/impl/EsEntityIndexImpl.java   | 29 ++--
 1 file changed, 20 insertions(+), 9 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/usergrid/blob/41702956/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
--
diff --git 
a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
 
b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
index f4fae2b..3d2f576 100644
--- 
a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
+++ 
b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
@@ -70,6 +70,7 @@ import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.index.query.*;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.indices.IndexAlreadyExistsException;
+import org.elasticsearch.indices.IndexMissingException;
 import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.search.SearchHits;
 import org.elasticsearch.search.aggregations.metrics.sum.Sum;
@@ -804,15 +805,25 @@ public class EsEntityIndexImpl implements 
EntityIndex,VersionedData {
 }
 
 private long getIndexSize(){
-final IndicesStatsResponse statsResponse = esProvider.getClient()
-.admin()
-.indices()
-.prepareStats(indexLocationStrategy.getIndexInitialName())
-.all()
-.execute()
-.actionGet();
-final CommonStats indexStats = 
statsResponse.getIndex(indexLocationStrategy.getIndexInitialName()).getTotal();
-return indexStats.getStore().getSizeInBytes();
+long indexSize = 0L;
+final String indexName = indexLocationStrategy.getIndexInitialName();
+try {
+final IndicesStatsResponse statsResponse = esProvider.getClient()
+.admin()
+.indices()
+.prepareStats(indexName)
+.all()
+.execute()
+.actionGet();
+final CommonStats indexStats = 
statsResponse.getIndex(indexName).getTotal();
+indexSize = indexStats.getStore().getSizeInBytes();
+} catch (IndexMissingException e) {
+// if for some reason the index size does not exist,
+// log an error and we can assume size is 0 as it doesn't exist
+logger.error("Unable to get size for index {} due to 
IndexMissingException for app {}",
+indexName, 
indexLocationStrategy.getApplicationScope().getApplication().getUuid());
+}
+return indexSize;
 }
 
 @Override

usergrid git commit: Initial commit for exposing collection re-index to non sysadmin users.

[mrusso]

Repository: usergrid
Updated Branches:
  refs/heads/expose-reindex [created] 148337ec8


Initial commit for exposing collection re-index to non sysadmin users.


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/148337ec
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/148337ec
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/148337ec

Branch: refs/heads/expose-reindex
Commit: 148337ec848b2077c14967a138242e39476b2ef6
Parents: a6fee78
Author: Michael Russo 
Authored: Wed Jun 28 11:07:15 2017 -0700
Committer: Michael Russo 
Committed: Wed Jun 28 11:07:15 2017 -0700

--
 .../corepersistence/index/ReIndexService.java   |  33 +-
 .../index/ReIndexServiceImpl.java   | 106 +--
 .../rest/applications/CollectionResource.java   |  28 +++--
 .../usergrid/rest/system/IndexResource.java |  78 --
 4 files changed, 194 insertions(+), 51 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/usergrid/blob/148337ec/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexService.java
--
diff --git 
a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexService.java
 
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexService.java
index b9238e5..d37f117 100644
--- 
a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexService.java
+++ 
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexService.java
@@ -20,6 +20,8 @@
 package org.apache.usergrid.corepersistence.index;
 
 
+import org.apache.usergrid.utils.StringUtils;
+
 /**
  * An interface for re-indexing all entities in an application
  */
@@ -47,6 +49,13 @@ public interface ReIndexService {
  */
 ReIndexStatus getStatus( final String jobId );
 
+/**
+ * Get the status of a collection job
+ * @param collectionName The collectionName for the rebuild index
+ * @return
+ */
+ReIndexStatus getStatusForCollection( final String appIdString, final 
String collectionName );
+
 
 /**
  * The response when requesting a re-index operation
@@ -56,14 +65,27 @@ public interface ReIndexService {
 final Status status;
 final long numberProcessed;
 final long lastUpdated;
+final String collectionName;
 
 
 public ReIndexStatus( final String jobId, final Status status, final 
long numberProcessed,
-  final long lastUpdated ) {
-this.jobId = jobId;
+  final long lastUpdated, final String 
collectionName ) {
+
+if(StringUtils.isNotEmpty(jobId)){
+this.jobId = jobId;
+}else {
+this.jobId = "";
+}
+
 this.status = status;
 this.numberProcessed = numberProcessed;
 this.lastUpdated = lastUpdated;
+
+if(StringUtils.isNotEmpty(collectionName)){
+this.collectionName = collectionName;
+}else {
+this.collectionName = "";
+}
 }
 
 
@@ -74,6 +96,13 @@ public interface ReIndexService {
 return jobId;
 }
 
+/**
+ * Get the jobId used to resume this operation
+ */
+public String getCollectionName() {
+return collectionName;
+}
+
 
 /**
  * Get the last updated time, as a long

http://git-wip-us.apache.org/repos/asf/usergrid/blob/148337ec/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexServiceImpl.java
--
diff --git 
a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexServiceImpl.java
 
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexServiceImpl.java
index 0660d5e..0731af7 100644
--- 
a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexServiceImpl.java
+++ 
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexServiceImpl.java
@@ -74,6 +74,7 @@ public class ReIndexServiceImpl implements ReIndexService {
 private static final String MAP_COUNT_KEY = "count";
 private static final String MAP_STATUS_KEY = "status";
 private static final String MAP_UPDATED_KEY = "lastUpdated";
+private static final String MAP_SEPARATOR = "|||";
 
 
 private final AllApplicationsObservable allApplicationsObservable;
@@ -139,7 +140,9 @@ public class ReIndexServiceImpl implements ReIndexService {
 
 // create an observable that loads a batch to be indexed
 
-if(reIndexRequestBuilder.getCollectionName().isPresent()) {
+final boolean isForColl

usergrid git commit: Converts de-indexing of edges and entities to identify documents in the index to delete based on data in Cassandra vs. resource intensive queries to the index. Fixes issue where n

[mrusso]

Repository: usergrid
Updated Branches:
  refs/heads/collectionDelete a881d5b55 -> 99ba349c8


Converts de-indexing of edges and entities to identify documents in the index 
to delete based on data in Cassandra vs. resource intensive queries to the 
index. Fixes issue where nodes were not actually getting deleted from graph as 
marked edges were being filtered out during the delete process itself. Update 
to a newer vs. of jamm (used for jvm memory management in the test framework).


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/99ba349c
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/99ba349c
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/99ba349c

Branch: refs/heads/collectionDelete
Commit: 99ba349c822ebb5f4d7599854b46ca2115b13b10
Parents: a881d5b
Author: Michael Russo 
Authored: Sat Jul 8 17:10:25 2017 -0700
Committer: Michael Russo 
Committed: Sat Jul 8 17:10:25 2017 -0700

--
 .../corepersistence/EntityManagerFig.java   |   2 +-
 .../asyncevents/AsyncEventServiceImpl.java  |  24 ++--
 .../asyncevents/EventBuilder.java   |  17 +--
 .../asyncevents/EventBuilderImpl.java   | 144 ++-
 .../corepersistence/index/IndexService.java |  16 +--
 .../corepersistence/index/IndexServiceImpl.java |  57 ++--
 .../read/traverse/AbstractReadGraphFilter.java  |  37 +
 .../AbstractReadReverseGraphFilter.java |  36 +
 .../rx/EdgesFromSourceObservableIT.java |   2 +-
 .../rx/EdgesToTargetObservableIT.java   |   4 +-
 .../core/src/test/resources/project.properties  |   2 +-
 .../persistence/graph/GraphManager.java |   2 +-
 .../graph/impl/GraphManagerImpl.java|  20 +--
 .../graph/impl/SimpleSearchByEdgeType.java  |   9 ++
 .../graph/impl/stage/NodeDeleteListener.java|   4 +-
 .../impl/stage/NodeDeleteListenerImpl.java  |  62 +++-
 .../graph/serialization/EdgesObservable.java|   6 +-
 .../impl/EdgeMetadataSerializationV2Impl.java   |   1 -
 .../serialization/impl/EdgesObservableImpl.java |  20 ++-
 .../impl/TargetIdObservableImpl.java|   4 +-
 .../impl/migration/EdgeDataMigrationImpl.java   |   2 +-
 .../graph/impl/NodeDeleteListenerTest.java  |   8 +-
 .../usergrid/persistence/index/EntityIndex.java |  12 +-
 .../index/impl/EsEntityIndexImpl.java   |  63 
 stack/pom.xml   |   8 +-
 stack/rest/pom.xml  |   4 +-
 .../rest/src/test/resources/project.properties  |   2 +-
 stack/services/pom.xml  |   4 +-
 .../src/test/resources/project.properties   |   2 +-
 stack/test-utils/pom.xml|   4 +-
 .../src/test/resources/project.properties   |   2 +-
 stack/tools/pom.xml |   2 +-
 stack/websocket/pom.xml |   2 +-
 33 files changed, 241 insertions(+), 343 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/usergrid/blob/99ba349c/stack/core/src/main/java/org/apache/usergrid/corepersistence/EntityManagerFig.java
--
diff --git 
a/stack/core/src/main/java/org/apache/usergrid/corepersistence/EntityManagerFig.java
 
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/EntityManagerFig.java
index 872ffbb..46c7a1d 100644
--- 
a/stack/core/src/main/java/org/apache/usergrid/corepersistence/EntityManagerFig.java
+++ 
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/EntityManagerFig.java
@@ -39,7 +39,7 @@ public interface EntityManagerFig extends GuicyFig {
 int sleep();
 
 @Key( "usergrid.entityManager.enable_deindex_on_update" )
-@Default( "true" )
+@Default( "false" )
 boolean getDeindexOnUpdate();
 
 /**

http://git-wip-us.apache.org/repos/asf/usergrid/blob/99ba349c/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventServiceImpl.java
--
diff --git 
a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventServiceImpl.java
 
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventServiceImpl.java
index 5628a11..257e172 100644
--- 
a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventServiceImpl.java
+++ 
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventServiceImpl.java
@@ -57,6 +57,7 @@ import org.apache.usergrid.persistence.model.entity.Id;
 import org.apache.usergrid.persistence.model.util.UUIDGenerator;
 import org.apache.usergrid.persistence.queue.*;
 import org.apache.usergrid.persistence.queue.impl.LegacyQueueScopeImpl;
+import org.apache.usergrid.pe

[6/7] usergrid git commit: Fix precondition checks and add another try/catch around the token workflow org id.

[mrusso]

Fix precondition checks and add another try/catch around the token workflow org 
id.


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/c0ba7103
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/c0ba7103
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/c0ba7103

Branch: refs/heads/master
Commit: c0ba7103949564dd9cebc3631fe8e5002a7e73d2
Parents: 9963449
Author: Michael Russo 
Authored: Mon Jun 19 08:49:57 2017 -0700
Committer: Michael Russo 
Committed: Mon Jun 19 08:49:57 2017 -0700

--
 .../persistence/token/impl/TokenSerializationImpl.java   | 8 
 .../usergrid/security/tokens/impl/TokenServiceImpl.java  | 6 +-
 2 files changed, 9 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/usergrid/blob/c0ba7103/stack/corepersistence/token/src/main/java/org/apache/usergrid/persistence/token/impl/TokenSerializationImpl.java
--
diff --git 
a/stack/corepersistence/token/src/main/java/org/apache/usergrid/persistence/token/impl/TokenSerializationImpl.java
 
b/stack/corepersistence/token/src/main/java/org/apache/usergrid/persistence/token/impl/TokenSerializationImpl.java
index 240da00..7825da0 100644
--- 
a/stack/corepersistence/token/src/main/java/org/apache/usergrid/persistence/token/impl/TokenSerializationImpl.java
+++ 
b/stack/corepersistence/token/src/main/java/org/apache/usergrid/persistence/token/impl/TokenSerializationImpl.java
@@ -192,9 +192,9 @@ public class TokenSerializationImpl implements 
TokenSerialization {
 public void updateTokenAccessTime(UUID tokenUUID, long accessedTime, long 
inactiveTime, int ttl ){
 
 Preconditions.checkNotNull(tokenUUID, "token UUID is required");
-Preconditions.checkNotNull(accessedTime, "accessedTime is required");
-Preconditions.checkNotNull(inactiveTime, "inactiveTime is required");
-Preconditions.checkNotNull(ttl, "ttl is required");
+Preconditions.checkArgument(accessedTime > -1 , "accessedTime is 
required to be positive");
+Preconditions.checkArgument(inactiveTime > -1 , "inactiveTime is 
required to be positive");
+Preconditions.checkArgument(ttl > -1 , "ttl is required to be 
positive");
 
 logger.trace("updateTokenAccessTime, token UUID: {}, accessedTime: {}, 
inactiveTime: {}, ttl: {}",
 tokenUUID, accessedTime, inactiveTime, ttl);
@@ -287,7 +287,7 @@ public class TokenSerializationImpl implements 
TokenSerialization {
 
 Preconditions.checkNotNull(tokenUUID, "tokenUUID is required");
 Preconditions.checkNotNull(tokenUUID, "tokenInfo is required");
-Preconditions.checkNotNull(ttl, "ttl is required");
+Preconditions.checkArgument(ttl > -1 , "ttl is required to be 
positive");
 
 logger.trace("putTokenInfo, token UUID: {}, tokenInfo: {}, ttl: {}", 
tokenUUID, tokenInfo, ttl);
 

http://git-wip-us.apache.org/repos/asf/usergrid/blob/c0ba7103/stack/services/src/main/java/org/apache/usergrid/security/tokens/impl/TokenServiceImpl.java
--
diff --git 
a/stack/services/src/main/java/org/apache/usergrid/security/tokens/impl/TokenServiceImpl.java
 
b/stack/services/src/main/java/org/apache/usergrid/security/tokens/impl/TokenServiceImpl.java
index f149a1f..05859ad 100644
--- 
a/stack/services/src/main/java/org/apache/usergrid/security/tokens/impl/TokenServiceImpl.java
+++ 
b/stack/services/src/main/java/org/apache/usergrid/security/tokens/impl/TokenServiceImpl.java
@@ -456,7 +456,11 @@ public class TokenServiceImpl implements TokenService {
 
 UUID workflowOrgId = null;
 if (tokenDetails.containsKey(TOKEN_WORKFLOW_ORG_ID)) {
-workflowOrgId = (UUID) tokenDetails.get(TOKEN_WORKFLOW_ORG_ID);
+try {
+workflowOrgId = (UUID) tokenDetails.get(TOKEN_WORKFLOW_ORG_ID);
+} catch (ClassCastException cce){
+logger.error("Unable to cast {} to primitive UUID type", 
TOKEN_WORKFLOW_ORG_ID);
+}
 }
 
 return new TokenInfo( uuid, type, created, accessed, inactive, 
duration, principal, state, workflowOrgId );

[3/7] usergrid git commit: Finish convert token service persistence to Datastax. Add tests, and rename the package for TokenServiceImpl to impl from cassandra.

[mrusso]

Finish convert token service persistence to Datastax. Add tests, and rename the 
package for TokenServiceImpl to impl from cassandra.


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/146e47d6
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/146e47d6
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/146e47d6

Branch: refs/heads/master
Commit: 146e47d668788d72c4fc54a61b7155f508b6
Parents: 179f12d
Author: Michael Russo 
Authored: Sat Jun 17 23:47:47 2017 -0700
Committer: Michael Russo 
Committed: Sat Jun 17 23:47:47 2017 -0700

--
 .../src/test/resources/usergrid-test.properties |   2 -
 stack/core/pom.xml  |   6 +
 .../usergrid/corepersistence/CoreModule.java|   2 +
 .../usergrid/corepersistence/CpSetup.java   |  11 -
 .../persistence/cassandra/CassandraService.java |   3 -
 .../map/impl/MapSerializationImpl.java  |  35 +-
 .../persistence/token/TokenSerialization.java   |  50 ++
 .../persistence/token/guice/TokenModule.java|  10 +-
 .../token/impl/TokenSerialization.java  |  50 --
 .../token/impl/TokenSerializationImpl.java  | 482 +--
 .../persistence/token/TestTokenModule.java  |  38 +
 .../token/TokenSerializationTest.java   | 202 +
 .../persistence/token/TokenTestModule.java  |  40 -
 .../resources/usergrid-standalone.properties|   2 -
 .../rest/management/ManagementResource.java |   4 +-
 .../rest/management/users/UserResource.java |   2 +-
 .../rest/management/users/UsersResource.java|   3 +-
 .../rest/management/ManagementResourceIT.java   |   8 +-
 .../security/sso/SSOProviderFactory.java|   2 +-
 .../tokens/cassandra/TokenServiceImpl.java  | 848 ---
 .../security/tokens/impl/TokenServiceImpl.java  | 731 
 .../resources/usergrid-services-context.xml |   3 +-
 .../security/tokens/TokenServiceIT.java |   2 +-
 .../test/resources/testApplicationContext.xml   |   2 +-
 24 files changed, 1253 insertions(+), 1285 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/usergrid/blob/146e47d6/stack/config/src/test/resources/usergrid-test.properties
--
diff --git a/stack/config/src/test/resources/usergrid-test.properties 
b/stack/config/src/test/resources/usergrid-test.properties
index 143d60a..5def24e 100644
--- a/stack/config/src/test/resources/usergrid-test.properties
+++ b/stack/config/src/test/resources/usergrid-test.properties
@@ -104,8 +104,6 @@ collection.uniquevalues.authoritative.region=us-east
 
 usergrid.counter.batch.interval=10
 #usergrid.auth.token_secret_salt=super secret token value
-#usergrid.auth.token_expires_from_last_use=false
-#usergrid.auth.token_refresh_reuses_id=false
 
 # max time to persist tokens for (milliseconds)
 #usergrid.auth.token.persist.expires=0

http://git-wip-us.apache.org/repos/asf/usergrid/blob/146e47d6/stack/core/pom.xml
--
diff --git a/stack/core/pom.xml b/stack/core/pom.xml
index 09047bb..658320e 100644
--- a/stack/core/pom.xml
+++ b/stack/core/pom.xml
@@ -428,6 +428,12 @@
 
 
 org.apache.usergrid
+token
+${project.version}
+
+
+
+org.apache.usergrid
 actorsystem
 ${project.version}
 

http://git-wip-us.apache.org/repos/asf/usergrid/blob/146e47d6/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
--
diff --git 
a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java 
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
index af297f2..ec6b775 100644
--- 
a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
+++ 
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
@@ -44,6 +44,7 @@ import 
org.apache.usergrid.persistence.core.scope.ApplicationScope;
 import org.apache.usergrid.persistence.graph.guice.GraphModule;
 import 
org.apache.usergrid.persistence.graph.serialization.impl.migration.GraphNode;
 import org.apache.usergrid.persistence.index.guice.IndexModule;
+import org.apache.usergrid.persistence.token.guice.TokenModule;
 import org.safehaus.guicyfig.GuicyFigModule;
 
 import java.util.Properties;
@@ -67,6 +68,7 @@ public class CoreModule extends AbstractModule {
 install( new CommonModule());
 install( new LockModule());
 install( new CacheModule());
+install( new TokenModule());
 install( new CollectionModule() {
 /**
  * configure our migration data provider for all entities in t

[7/7] usergrid git commit: Merge commit 'refs/pull/573/head' of github.com:apache/usergrid

[mrusso]

Merge commit 'refs/pull/573/head' of github.com:apache/usergrid


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/375814e0
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/375814e0
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/375814e0

Branch: refs/heads/master
Commit: 375814e00330f28b4209c7595ed591187abbb245
Parents: d8a7e10 c0ba710
Author: Michael Russo 
Authored: Mon Jun 19 12:58:30 2017 -0700
Committer: Michael Russo 
Committed: Mon Jun 19 12:58:30 2017 -0700

--
 .../src/test/resources/usergrid-test.properties |   2 -
 stack/core/pom.xml  |   6 +
 .../usergrid/corepersistence/CoreModule.java|   3 +-
 .../usergrid/corepersistence/CpSetup.java   |  11 -
 .../persistence/cassandra/CassandraService.java |   3 -
 .../map/impl/MapSerializationImpl.java  |  35 +-
 stack/corepersistence/pom.xml   |   1 +
 stack/corepersistence/token/pom.xml |  70 ++
 .../persistence/token/TokenSerialization.java   |  50 ++
 .../persistence/token/guice/TokenModule.java|  44 +
 .../token/impl/TokenSerializationImpl.java  | 451 ++
 .../persistence/token/TestTokenModule.java  |  38 +
 .../token/TokenSerializationTest.java   | 202 +
 .../token/src/test/resources/log4j.properties   |  38 +
 .../resources/usergrid-standalone.properties|   2 -
 .../rest/management/ManagementResource.java |   4 +-
 .../rest/management/users/UserResource.java |   2 +-
 .../rest/management/users/UsersResource.java|   3 +-
 .../rest/management/ManagementResourceIT.java   |   8 +-
 .../security/sso/SSOProviderFactory.java|   2 +-
 .../tokens/cassandra/TokenServiceImpl.java  | 848 ---
 .../security/tokens/impl/TokenServiceImpl.java  | 748 
 .../notifications/InactiveDeviceManager.java|   6 -
 .../resources/usergrid-services-context.xml |   3 +-
 .../security/tokens/TokenServiceIT.java |   2 +-
 .../test/resources/testApplicationContext.xml   |   2 +-
 26 files changed, 1680 insertions(+), 904 deletions(-)
--

[1/7] usergrid git commit: Initial commit of new token module to replace Hector for access token storage.

[mrusso]

Repository: usergrid
Updated Branches:
  refs/heads/master d8a7e10ca -> 375814e00


Initial commit of new token module to replace Hector for access token storage.


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/179f12db
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/179f12db
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/179f12db

Branch: refs/heads/master
Commit: 179f12db0ef55cf9c28936678a933e2de04034eb
Parents: 3f819dc
Author: Michael Russo 
Authored: Fri Jun 16 00:15:14 2017 -0700
Committer: Michael Russo 
Committed: Fri Jun 16 00:15:14 2017 -0700

--
 stack/corepersistence/pom.xml   |   1 +
 stack/corepersistence/token/pom.xml |  70 +++
 .../persistence/token/guice/TokenModule.java|  48 ++
 .../token/impl/TokenSerialization.java  |  50 ++
 .../token/impl/TokenSerializationImpl.java  | 536 +++
 .../persistence/token/TokenTestModule.java  |  40 ++
 .../token/src/test/resources/log4j.properties   |  38 ++
 7 files changed, 783 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/usergrid/blob/179f12db/stack/corepersistence/pom.xml
--
diff --git a/stack/corepersistence/pom.xml b/stack/corepersistence/pom.xml
index 8a45323..45aad4b 100644
--- a/stack/corepersistence/pom.xml
+++ b/stack/corepersistence/pom.xml
@@ -113,6 +113,7 @@ limitations under the License.
 queue
 cache
 actorsystem
+token
 
 
 

http://git-wip-us.apache.org/repos/asf/usergrid/blob/179f12db/stack/corepersistence/token/pom.xml
--
diff --git a/stack/corepersistence/token/pom.xml 
b/stack/corepersistence/token/pom.xml
new file mode 100644
index 000..fcbd73c
--- /dev/null
+++ b/stack/corepersistence/token/pom.xml
@@ -0,0 +1,70 @@
+
+
+http://maven.apache.org/POM/4.0.0";
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 
http://maven.apache.org/xsd/maven-4.0.0.xsd";>
+
+
+persistence
+org.apache.usergrid
+2.2.0-SNAPSHOT
+
+
+4.0.0
+The module for handling access token persistence
+
+token
+Usergrid Token
+
+
+
+
+org.apache.commons
+commons-lang3
+${commons.lang.version}
+
+
+
+org.apache.usergrid
+common
+${project.version}
+
+
+
+
+
+org.apache.usergrid
+common
+${project.version}
+tests
+test
+
+
+
+
+
+
+

http://git-wip-us.apache.org/repos/asf/usergrid/blob/179f12db/stack/corepersistence/token/src/main/java/org/apache/usergrid/persistence/token/guice/TokenModule.java
--
diff --git 
a/stack/corepersistence/token/src/main/java/org/apache/usergrid/persistence/token/guice/TokenModule.java
 
b/stack/corepersistence/token/src/main/java/org/apache/usergrid/persistence/token/guice/TokenModule.java
new file mode 100644
index 000..13d6b43
--- /dev/null
+++ 
b/stack/corepersistence/token/src/main/java/org/apache/usergrid/persistence/token/guice/TokenModule.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ *  contributor license agreements.  The ASF licenses this file to You
+ * under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.  For additional information regarding
+ * copyright in this work, please see the NOTICE file in the top level
+ * directory of this distribution.
+ */
+package org.apache.usergrid.persistence.token.guice;
+
+import com.google.inject.AbstractModule;
+import com.google.inject.Key;
+import com.google.inject.multibindings.Multibinder;
+import org.apache.usergrid.persistence.cache.CacheFactory;
+import org.apache.usergrid.persistence.cache.impl.CacheFactoryImpl;
+import org.apache.usergrid.persistence.cache.impl.TokenSerialization;
+import org.apache.usergrid.persistence.cache.impl.TokenSerializationImpl;
+import org.apache.usergrid.persistence.core.migration.schema.Migration;
+
+
+/**
+ * Wire up cache impl.
+ */
+

[2/7] usergrid git commit: Finish convert token service persistence to Datastax. Add tests, and rename the package for TokenServiceImpl to impl from cassandra.

[mrusso]

http://git-wip-us.apache.org/repos/asf/usergrid/blob/146e47d6/stack/services/src/main/java/org/apache/usergrid/security/tokens/cassandra/TokenServiceImpl.java
--
diff --git 
a/stack/services/src/main/java/org/apache/usergrid/security/tokens/cassandra/TokenServiceImpl.java
 
b/stack/services/src/main/java/org/apache/usergrid/security/tokens/cassandra/TokenServiceImpl.java
deleted file mode 100644
index 6ea6de0..000
--- 
a/stack/services/src/main/java/org/apache/usergrid/security/tokens/cassandra/TokenServiceImpl.java
+++ /dev/null
@@ -1,848 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *  http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.usergrid.security.tokens.cassandra;
-
-
-import com.google.inject.Injector;
-import me.prettyprint.hector.api.Keyspace;
-import me.prettyprint.hector.api.beans.HColumn;
-import me.prettyprint.hector.api.mutation.Mutator;
-import org.apache.usergrid.corepersistence.CpEntityManagerFactory;
-import org.apache.usergrid.corepersistence.util.CpNamingUtils;
-import org.apache.usergrid.management.ApplicationCreator;
-import org.apache.usergrid.management.ManagementService;
-import org.apache.usergrid.management.UserInfo;
-import org.apache.usergrid.persistence.EntityManagerFactory;
-import org.apache.usergrid.persistence.cassandra.CassandraService;
-import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
-import org.apache.usergrid.persistence.entities.Application;
-import org.apache.usergrid.security.AuthPrincipalInfo;
-import org.apache.usergrid.security.AuthPrincipalType;
-import org.apache.usergrid.security.sso.SSOProviderFactory;
-import org.apache.usergrid.security.tokens.TokenCategory;
-import org.apache.usergrid.security.tokens.TokenInfo;
-import org.apache.usergrid.security.tokens.TokenService;
-import org.apache.usergrid.security.tokens.exceptions.BadTokenException;
-import org.apache.usergrid.security.tokens.exceptions.ExpiredTokenException;
-import org.apache.usergrid.security.tokens.exceptions.InvalidTokenException;
-import org.apache.usergrid.security.sso.ExternalSSOProvider;
-import org.apache.usergrid.utils.ConversionUtils;
-import org.apache.usergrid.utils.JsonUtils;
-import org.apache.usergrid.utils.UUIDUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.beans.factory.annotation.Qualifier;
-import org.springframework.util.Assert;
-
-import javax.ws.rs.client.Client;
-import java.nio.ByteBuffer;
-import java.util.*;
-
-import static java.lang.System.currentTimeMillis;
-import static me.prettyprint.hector.api.factory.HFactory.createColumn;
-import static me.prettyprint.hector.api.factory.HFactory.createMutator;
-import static org.apache.commons.codec.binary.Base64.decodeBase64;
-import static org.apache.commons.codec.binary.Base64.encodeBase64URLSafeString;
-import static org.apache.commons.codec.digest.DigestUtils.sha;
-import static 
org.apache.usergrid.persistence.cassandra.CassandraPersistenceUtils.getColumnMap;
-import static 
org.apache.usergrid.persistence.cassandra.CassandraService.PRINCIPAL_TOKEN_CF;
-import static 
org.apache.usergrid.persistence.cassandra.CassandraService.TOKENS_CF;
-import static org.apache.usergrid.persistence.cassandra.Serializers.*;
-import static org.apache.usergrid.security.AuthPrincipalType.ADMIN_USER;
-import static org.apache.usergrid.security.tokens.TokenCategory.*;
-import static org.apache.usergrid.utils.ConversionUtils.*;
-import static org.apache.usergrid.utils.MapUtils.hasKeys;
-import static org.apache.usergrid.utils.MapUtils.hashMap;
-import static org.apache.usergrid.utils.UUIDUtils.getTimestampInMillis;
-
-
-public class TokenServiceImpl implements TokenService {
-
-private static final Logger logger = LoggerFactory.getLogger( 
TokenServiceImpl.class );
-
-public static final String PROPERTIES_AUTH_TOKEN_SECRET_SALT = 
"usergrid.auth.token_secret_salt";
-public static final String PROPERTIES_AUTH_TOKEN_EXPIRES_FROM_LAST_USE =
-"usergrid.auth.token_expires_from_last_use";
-public static final String PROPERTIES_AUTH_TOKEN_REFRESH_REUSES_ID = 
"usergrid.auth.token_refresh_reu

[4/7] usergrid git commit: Fix issue where tokenInfo was not being updated with latest inactive value. Add trace logging to TokenSerializationImpl. Fix issue with push notifications where the Inactive

[mrusso]

Fix issue where tokenInfo was not being updated with latest inactive value. Add 
trace logging to TokenSerializationImpl. Fix issue with push notifications 
where the InactiveDeviceManager was making a duplicate (yet incorrect) query 
for devices.


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/bea9a418
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/bea9a418
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/bea9a418

Branch: refs/heads/master
Commit: bea9a4180ebadb79e7f101cfc6d0d06efc39b359
Parents: 146e47d
Author: Michael Russo 
Authored: Sun Jun 18 12:36:56 2017 -0700
Committer: Michael Russo 
Committed: Sun Jun 18 12:36:56 2017 -0700

--
 .../token/impl/TokenSerializationImpl.java  | 35 ++--
 .../security/tokens/impl/TokenServiceImpl.java  |  9 +++--
 .../notifications/InactiveDeviceManager.java|  6 
 3 files changed, 31 insertions(+), 19 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/usergrid/blob/bea9a418/stack/corepersistence/token/src/main/java/org/apache/usergrid/persistence/token/impl/TokenSerializationImpl.java
--
diff --git 
a/stack/corepersistence/token/src/main/java/org/apache/usergrid/persistence/token/impl/TokenSerializationImpl.java
 
b/stack/corepersistence/token/src/main/java/org/apache/usergrid/persistence/token/impl/TokenSerializationImpl.java
index 2207b99..240da00 100644
--- 
a/stack/corepersistence/token/src/main/java/org/apache/usergrid/persistence/token/impl/TokenSerializationImpl.java
+++ 
b/stack/corepersistence/token/src/main/java/org/apache/usergrid/persistence/token/impl/TokenSerializationImpl.java
@@ -130,22 +130,27 @@ public class TokenSerializationImpl implements 
TokenSerialization {
 public void deleteTokens(final List tokenUUIDs, final ByteBuffer 
principalKeyBuffer){
 
 Preconditions.checkNotNull(tokenUUIDs, "token UUID list is required");
+Preconditions.checkNotNull(tokenUUIDs, "principalKeyBuffer is 
required");
 
-final BatchStatement batchStatement = new BatchStatement();
+logger.trace("deleteTokens, token UUIDs: {}", tokenUUIDs);
 
-tokenUUIDs.forEach( tokenUUID -> batchStatement.add(
-QueryBuilder.delete()
-.from(TOKENS_TABLE)
-.where(QueryBuilder
-.eq("key", DataType.uuid().serialize(tokenUUID, 
ProtocolVersion.NEWEST_SUPPORTED);
+final BatchStatement batchStatement = new BatchStatement();
 
-if(principalKeyBuffer != null){
+tokenUUIDs.forEach( tokenUUID ->
 batchStatement.add(
 QueryBuilder.delete()
-.from(PRINCIPAL_TOKENS_TABLE)
+.from(TOKENS_TABLE)
 .where(QueryBuilder
-.eq("key", principalKeyBuffer)));
-}
+.eq("key", DataType.uuid().serialize(tokenUUID, 
ProtocolVersion.NEWEST_SUPPORTED)))
+)
+);
+
+batchStatement.add(
+QueryBuilder.delete()
+.from(PRINCIPAL_TOKENS_TABLE)
+.where(QueryBuilder
+.eq("key", principalKeyBuffer)));
+
 
 session.execute(batchStatement);
 
@@ -157,6 +162,9 @@ public class TokenSerializationImpl implements 
TokenSerialization {
 
 Preconditions.checkNotNull(tokenUUID, "token UUID is required");
 
+logger.trace("revokeToken, token UUID: {}", tokenUUID);
+
+
 final BatchStatement batchStatement = new BatchStatement();
 
 batchStatement.add(
@@ -188,6 +196,8 @@ public class TokenSerializationImpl implements 
TokenSerialization {
 Preconditions.checkNotNull(inactiveTime, "inactiveTime is required");
 Preconditions.checkNotNull(ttl, "ttl is required");
 
+logger.trace("updateTokenAccessTime, token UUID: {}, accessedTime: {}, 
inactiveTime: {}, ttl: {}",
+tokenUUID, accessedTime, inactiveTime, ttl);
 
 final BatchStatement batchStatement = new BatchStatement();
 final Clause inKey =
@@ -265,6 +275,8 @@ public class TokenSerializationImpl implements 
TokenSerialization {
 
 });
 
+logger.trace("getTokenInfo, info: {}", tokenInfo);
+
 return tokenInfo;
 }
 
@@ -277,6 +289,7 @@ public class TokenSerializationImpl implements 
TokenSerialization {
 Preconditions.checkNotNull(tokenUUID, "tokenInfo is required");
 Preconditions.checkNotNull(ttl, "ttl is required");
 
+logger.trace("putTokenInfo, token UUID: {}, tokenInfo: {}, ttl: {}", 
tokenUUID, tokenInfo, ttl);
 
 final BatchStatement batchStatement = new BatchStatement();
 final Using usingTTL = QueryBuilder.ttl(ttl);
@@ -332,6 +345,8 @@ public class 

[5/7] usergrid git commit: Add try/catch around fetching and casting of token info from serialization map. Also, throw exceptions early.

[mrusso]

Add try/catch around fetching and casting of token info from serialization map. 
Also, throw exceptions early.


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/99634490
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/99634490
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/99634490

Branch: refs/heads/master
Commit: 9963449003b1559277780ab193fb529ddfdf3b4b
Parents: bea9a41
Author: Michael Russo 
Authored: Mon Jun 19 08:31:31 2017 -0700
Committer: Michael Russo 
Committed: Mon Jun 19 08:31:31 2017 -0700

--
 .../security/tokens/impl/TokenServiceImpl.java  | 20 +++-
 1 file changed, 15 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/usergrid/blob/99634490/stack/services/src/main/java/org/apache/usergrid/security/tokens/impl/TokenServiceImpl.java
--
diff --git 
a/stack/services/src/main/java/org/apache/usergrid/security/tokens/impl/TokenServiceImpl.java
 
b/stack/services/src/main/java/org/apache/usergrid/security/tokens/impl/TokenServiceImpl.java
index 21bd0db..f149a1f 100644
--- 
a/stack/services/src/main/java/org/apache/usergrid/security/tokens/impl/TokenServiceImpl.java
+++ 
b/stack/services/src/main/java/org/apache/usergrid/security/tokens/impl/TokenServiceImpl.java
@@ -417,11 +417,21 @@ public class TokenServiceImpl implements TokenService {
 throw new InvalidTokenException( "Token not found in database" );
 }
 
-String type = (String) tokenDetails.get(TOKEN_TYPE);
-long created = (long) tokenDetails.get(TOKEN_CREATED);
-long accessed = (long) tokenDetails.get(TOKEN_ACCESSED);
-long inactive = (long) tokenDetails.get(TOKEN_INACTIVE);
-long duration = (long) tokenDetails.get(TOKEN_DURATION);
+String type;
+long created, accessed, inactive, duration;
+try {
+type = (String) tokenDetails.get(TOKEN_TYPE);
+created = (long) tokenDetails.get(TOKEN_CREATED);
+accessed = (long) tokenDetails.get(TOKEN_ACCESSED);
+inactive = (long) tokenDetails.get(TOKEN_INACTIVE);
+duration = (long) tokenDetails.get(TOKEN_DURATION);
+} catch (ClassCastException cce){
+logger.error("Unable to cast token info to primitive type: {}", 
cce);
+throw new RuntimeException("Unable to cast token info to primitive 
type");
+} catch (NullPointerException npe){
+logger.error("Unable to obtain token info from serialization 
layer, on or more missing properties: {}", npe);
+throw new RuntimeException("Unable to obtain token info from 
serialization layer, on or more missing properties");
+}
 
 String principalTypeStr = (String) 
tokenDetails.get(TOKEN_PRINCIPAL_TYPE);
 

usergrid git commit: Remove unused import.

[mrusso]

Repository: usergrid
Updated Branches:
  refs/heads/master c748242fa -> d8a7e10ca


Remove unused import.


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/d8a7e10c
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/d8a7e10c
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/d8a7e10c

Branch: refs/heads/master
Commit: d8a7e10cab3b6403d4688b4256c0449652261b62
Parents: c748242
Author: Michael Russo 
Authored: Mon Jun 19 12:56:35 2017 -0700
Committer: Michael Russo 
Committed: Mon Jun 19 12:56:35 2017 -0700

--
 .../apache/usergrid/persistence/queue/impl/SNSQueueManagerImpl.java | 1 -
 1 file changed, 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/usergrid/blob/d8a7e10c/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/impl/SNSQueueManagerImpl.java
--
diff --git 
a/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/impl/SNSQueueManagerImpl.java
 
b/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/impl/SNSQueueManagerImpl.java
index 1370298..5b49bc7 100644
--- 
a/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/impl/SNSQueueManagerImpl.java
+++ 
b/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/impl/SNSQueueManagerImpl.java
@@ -26,7 +26,6 @@ import java.util.concurrent.ExecutorService;
 
 import com.amazonaws.ClientConfiguration;
 import com.amazonaws.services.sqs.model.*;
-import com.sun.javaws.exceptions.InvalidArgumentException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 

[1/2] usergrid git commit: dead letter queue handling 1. Add workers that move messages from dead letter queues back to the indexing and utility queues. 2. Change ERROR for DuplicateUniquePropertyExis

[mrusso]

Repository: usergrid
Updated Branches:
  refs/heads/master 3f819dc06 -> c748242fa


dead letter queue handling
1. Add workers that move messages from dead letter queues back to the indexing 
and utility queues.
2. Change ERROR for DuplicateUniquePropertyExistsException to INFO. This 
happens when someone tries to create an entity with the same unique value as 
another entity, which is not an error that should be logged.
3. Add better logging when cluster region or region list has an invalid region.


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/59354608
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/59354608
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/59354608

Branch: refs/heads/master
Commit: 5935460829622ceee91d72ff8a3b44529875ae29
Parents: 3f819dc
Author: Mike Dunker 
Authored: Wed Jun 14 16:00:13 2017 -0700
Committer: Mike Dunker 
Committed: Wed Jun 14 16:00:13 2017 -0700

--
 .../main/resources/usergrid-default.properties  |  26 +-
 .../usergrid/corepersistence/CoreModule.java|   1 +
 .../asyncevents/AsyncEventServiceImpl.java  | 283 +++
 .../index/IndexProcessorFig.java|  30 +-
 .../persistence/queue/LegacyQueueFig.java   |   7 +-
 .../persistence/queue/LegacyQueueManager.java   |   9 +
 .../persistence/queue/LegacyQueueScope.java |   5 +
 .../persistence/queue/LocalQueueManager.java|  20 +-
 .../persistence/queue/guice/QueueModule.java|   4 +
 .../queue/impl/LegacyQueueScopeImpl.java|  25 +-
 .../queue/impl/QakkaQueueManager.java   |  13 +
 .../queue/impl/SNSQueueManagerImpl.java |  97 +--
 .../services/AbstractCollectionService.java |   9 +
 .../services/queues/ImportQueueManager.java |   8 +
 14 files changed, 452 insertions(+), 85 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/usergrid/blob/59354608/stack/config/src/main/resources/usergrid-default.properties
--
diff --git a/stack/config/src/main/resources/usergrid-default.properties 
b/stack/config/src/main/resources/usergrid-default.properties
index 0d1a193..d505a96 100644
--- a/stack/config/src/main/resources/usergrid-default.properties
+++ b/stack/config/src/main/resources/usergrid-default.properties
@@ -305,10 +305,15 @@ cassandra.lock.writecl=LOCAL_QUORUM
 #
 #elasticsearch.refresh_search_max=10
 
-# Set the amount of time to wait when Elasticsearch rejects a requests before
+# Set the amount of time to wait when indexing or utility queue rejects a 
request before
 # retrying.  This provides simple backpressure. (in milliseconds)
 #
-#elasticsearch.rejected_retry_wait
+#elasticsearch.rejected_retry_wait=1000
+
+# Set the amount of time to wait when indexing or utility dead letter queue 
rejects a request before
+# retrying.  This provides simple backpressure. (in milliseconds)
+#
+#elasticsearch.deadletter.rejected_retry_wait=2000
 
 
 
@@ -332,18 +337,29 @@ cassandra.lock.writecl=LOCAL_QUORUM
 #
 #usergrid.use.default.queue=false
 
-# The number of worker threads used to read index write requests from the 
queue.
+# The number of worker threads used to read index write requests from the 
indexing queue.
 #
 #elasticsearch.worker_count=8
 
+# The number of worker threads used to read index write requests from the 
utility queue.
+#
+#elasticsearch.worker_count_utility=2
+
+# The number of worker threads used to read dead letter messages from the 
indexing dead letter queue.
+#
+#elasticsearch.worker_count_deadletter=1
+
+# The number of worker threads used to read dead letter messages from the 
utility dead letter queue.
+#
+#elasticsearch.worker_count_utility_deadletter=1
+
 # Set the number of worker threads used for processing index write requests to
 # Elasticsearch from the buffer.
 #
 #index.flush.workers=10
 
 # Set the implementation to use for queuing in Usergrid.
-# Valid values: TEST, LOCAL, SQS, SNS
-# NOTE: SQS and SNS equate to the same implementation of Amazon queue services.
+# Valid values: LOCAL, DISTRIBUTED, DISTRIBUTED_SNS
 #
 #elasticsearch.queue_impl=LOCAL
 

http://git-wip-us.apache.org/repos/asf/usergrid/blob/59354608/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
--
diff --git 
a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java 
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
index af297f2..d4cf8cb 100644
--- 
a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
+++ 
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
@@ -44,6 +44,7 @@ import 
org.apache.usergrid.persistence.core.scope.ApplicationScope;
 

[2/2] usergrid git commit: 1. Remove messages from dead letter queue if the message map entry was never received after a configurable timeout (15 minutes currently). 2. Log messages as they are moved

[mrusso]

1. Remove messages from dead letter queue if the message map entry was never 
received after a configurable timeout (15 minutes currently).
2. Log messages as they are moved from the dead letter queue back to the 
originating queue. Also log messages that are in dead letter queue and can't be 
moved to originating queue for some reason.


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/c748242f
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/c748242f
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/c748242f

Branch: refs/heads/master
Commit: c748242fa6a9bc79ebb60b4071cc532ab798fb3c
Parents: 5935460
Author: Mike Dunker 
Authored: Mon Jun 19 12:18:34 2017 -0700
Committer: Mike Dunker 
Committed: Mon Jun 19 12:18:34 2017 -0700

--
 .../asyncevents/AsyncEventServiceImpl.java  | 37 +---
 .../persistence/queue/LegacyQueueFig.java   |  3 ++
 2 files changed, 35 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/usergrid/blob/c748242f/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventServiceImpl.java
--
diff --git 
a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventServiceImpl.java
 
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventServiceImpl.java
index 7c33969..530cf7d 100644
--- 
a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventServiceImpl.java
+++ 
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventServiceImpl.java
@@ -724,10 +724,13 @@ public class AsyncEventServiceImpl implements 
AsyncEventService {
indexOperationMessage =
ObjectJsonSerializer.INSTANCE.fromString(highConsistency, 
IndexOperationMessage.class);
 
+   } else if (System.currentTimeMillis() > 
elasticsearchIndexEvent.getCreationTime() + queueFig.getMapMessageTimeout()) {
+// if esMapPersistence message hasn't been received yet, log 
and return (will be acked)
+logger.error("ES map message never received, removing message 
from queue. indexBatchId={}", messageId);
+return;
} else {
-
-   throw new 
IndexDocNotFoundException(elasticsearchIndexEvent.getIndexBatchId());
-
+logger.warn("ES map message not received yet. indexBatchId={} 
elapsedTimeMsec={}", messageId, System.currentTimeMillis() - 
elasticsearchIndexEvent.getCreationTime());
+throw new 
IndexDocNotFoundException(elasticsearchIndexEvent.getIndexBatchId());
}
 
 } else {
@@ -1105,14 +1108,38 @@ public class AsyncEventServiceImpl implements 
AsyncEventService {
  try {
  // put the dead letter 
messages back in the appropriate queue
  LegacyQueueManager 
returnQueue = null;
+ String queueType;
  if (isUtilityDeadQueue) {
- logger.warn("Utility 
dead queue message count: {}", messages.size());
  returnQueue = 
utilityQueue;
+ queueType = "utility";
  } else {
- logger.warn("Index 
dead queue message count: {}", messages.size());
  returnQueue = 
indexQueue;
+ queueType = "index";
  }
  List 
successMessages = returnQueue.sendQueueMessages(messages);
+ for (LegacyQueueMessage 
msg : successMessages) {
+ 
logger.warn("Returning message to {} queue: type:{}, messageId:{} body: {}", 
queueType, msg.getType(), msg.getMessageId(), msg.getStringBody());
+ }
+ int 
unsuccessfulMessagesSize = messages.size() - successMessages.size();
+ if 
(unsuccessfulMessagesSize > 0) {
+ // some messages 
couldn't be sent to originating queue, log
+   

usergrid git commit: Fix precondition checks and add another try/catch around the token workflow org id.

[mrusso]

Repository: usergrid
Updated Branches:
  refs/heads/token-service-to-dstax 996344900 -> c0ba71039


Fix precondition checks and add another try/catch around the token workflow org 
id.


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/c0ba7103
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/c0ba7103
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/c0ba7103

Branch: refs/heads/token-service-to-dstax
Commit: c0ba7103949564dd9cebc3631fe8e5002a7e73d2
Parents: 9963449
Author: Michael Russo 
Authored: Mon Jun 19 08:49:57 2017 -0700
Committer: Michael Russo 
Committed: Mon Jun 19 08:49:57 2017 -0700

--
 .../persistence/token/impl/TokenSerializationImpl.java   | 8 
 .../usergrid/security/tokens/impl/TokenServiceImpl.java  | 6 +-
 2 files changed, 9 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/usergrid/blob/c0ba7103/stack/corepersistence/token/src/main/java/org/apache/usergrid/persistence/token/impl/TokenSerializationImpl.java
--
diff --git 
a/stack/corepersistence/token/src/main/java/org/apache/usergrid/persistence/token/impl/TokenSerializationImpl.java
 
b/stack/corepersistence/token/src/main/java/org/apache/usergrid/persistence/token/impl/TokenSerializationImpl.java
index 240da00..7825da0 100644
--- 
a/stack/corepersistence/token/src/main/java/org/apache/usergrid/persistence/token/impl/TokenSerializationImpl.java
+++ 
b/stack/corepersistence/token/src/main/java/org/apache/usergrid/persistence/token/impl/TokenSerializationImpl.java
@@ -192,9 +192,9 @@ public class TokenSerializationImpl implements 
TokenSerialization {
 public void updateTokenAccessTime(UUID tokenUUID, long accessedTime, long 
inactiveTime, int ttl ){
 
 Preconditions.checkNotNull(tokenUUID, "token UUID is required");
-Preconditions.checkNotNull(accessedTime, "accessedTime is required");
-Preconditions.checkNotNull(inactiveTime, "inactiveTime is required");
-Preconditions.checkNotNull(ttl, "ttl is required");
+Preconditions.checkArgument(accessedTime > -1 , "accessedTime is 
required to be positive");
+Preconditions.checkArgument(inactiveTime > -1 , "inactiveTime is 
required to be positive");
+Preconditions.checkArgument(ttl > -1 , "ttl is required to be 
positive");
 
 logger.trace("updateTokenAccessTime, token UUID: {}, accessedTime: {}, 
inactiveTime: {}, ttl: {}",
 tokenUUID, accessedTime, inactiveTime, ttl);
@@ -287,7 +287,7 @@ public class TokenSerializationImpl implements 
TokenSerialization {
 
 Preconditions.checkNotNull(tokenUUID, "tokenUUID is required");
 Preconditions.checkNotNull(tokenUUID, "tokenInfo is required");
-Preconditions.checkNotNull(ttl, "ttl is required");
+Preconditions.checkArgument(ttl > -1 , "ttl is required to be 
positive");
 
 logger.trace("putTokenInfo, token UUID: {}, tokenInfo: {}, ttl: {}", 
tokenUUID, tokenInfo, ttl);
 

http://git-wip-us.apache.org/repos/asf/usergrid/blob/c0ba7103/stack/services/src/main/java/org/apache/usergrid/security/tokens/impl/TokenServiceImpl.java
--
diff --git 
a/stack/services/src/main/java/org/apache/usergrid/security/tokens/impl/TokenServiceImpl.java
 
b/stack/services/src/main/java/org/apache/usergrid/security/tokens/impl/TokenServiceImpl.java
index f149a1f..05859ad 100644
--- 
a/stack/services/src/main/java/org/apache/usergrid/security/tokens/impl/TokenServiceImpl.java
+++ 
b/stack/services/src/main/java/org/apache/usergrid/security/tokens/impl/TokenServiceImpl.java
@@ -456,7 +456,11 @@ public class TokenServiceImpl implements TokenService {
 
 UUID workflowOrgId = null;
 if (tokenDetails.containsKey(TOKEN_WORKFLOW_ORG_ID)) {
-workflowOrgId = (UUID) tokenDetails.get(TOKEN_WORKFLOW_ORG_ID);
+try {
+workflowOrgId = (UUID) tokenDetails.get(TOKEN_WORKFLOW_ORG_ID);
+} catch (ClassCastException cce){
+logger.error("Unable to cast {} to primitive UUID type", 
TOKEN_WORKFLOW_ORG_ID);
+}
 }
 
 return new TokenInfo( uuid, type, created, accessed, inactive, 
duration, principal, state, workflowOrgId );

usergrid git commit: Add try/catch around fetching and casting of token info from serialization map. Also, throw exceptions early.

[mrusso]

Repository: usergrid
Updated Branches:
  refs/heads/token-service-to-dstax bea9a4180 -> 996344900


Add try/catch around fetching and casting of token info from serialization map. 
Also, throw exceptions early.


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/99634490
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/99634490
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/99634490

Branch: refs/heads/token-service-to-dstax
Commit: 9963449003b1559277780ab193fb529ddfdf3b4b
Parents: bea9a41
Author: Michael Russo 
Authored: Mon Jun 19 08:31:31 2017 -0700
Committer: Michael Russo 
Committed: Mon Jun 19 08:31:31 2017 -0700

--
 .../security/tokens/impl/TokenServiceImpl.java  | 20 +++-
 1 file changed, 15 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/usergrid/blob/99634490/stack/services/src/main/java/org/apache/usergrid/security/tokens/impl/TokenServiceImpl.java
--
diff --git 
a/stack/services/src/main/java/org/apache/usergrid/security/tokens/impl/TokenServiceImpl.java
 
b/stack/services/src/main/java/org/apache/usergrid/security/tokens/impl/TokenServiceImpl.java
index 21bd0db..f149a1f 100644
--- 
a/stack/services/src/main/java/org/apache/usergrid/security/tokens/impl/TokenServiceImpl.java
+++ 
b/stack/services/src/main/java/org/apache/usergrid/security/tokens/impl/TokenServiceImpl.java
@@ -417,11 +417,21 @@ public class TokenServiceImpl implements TokenService {
 throw new InvalidTokenException( "Token not found in database" );
 }
 
-String type = (String) tokenDetails.get(TOKEN_TYPE);
-long created = (long) tokenDetails.get(TOKEN_CREATED);
-long accessed = (long) tokenDetails.get(TOKEN_ACCESSED);
-long inactive = (long) tokenDetails.get(TOKEN_INACTIVE);
-long duration = (long) tokenDetails.get(TOKEN_DURATION);
+String type;
+long created, accessed, inactive, duration;
+try {
+type = (String) tokenDetails.get(TOKEN_TYPE);
+created = (long) tokenDetails.get(TOKEN_CREATED);
+accessed = (long) tokenDetails.get(TOKEN_ACCESSED);
+inactive = (long) tokenDetails.get(TOKEN_INACTIVE);
+duration = (long) tokenDetails.get(TOKEN_DURATION);
+} catch (ClassCastException cce){
+logger.error("Unable to cast token info to primitive type: {}", 
cce);
+throw new RuntimeException("Unable to cast token info to primitive 
type");
+} catch (NullPointerException npe){
+logger.error("Unable to obtain token info from serialization 
layer, on or more missing properties: {}", npe);
+throw new RuntimeException("Unable to obtain token info from 
serialization layer, on or more missing properties");
+}
 
 String principalTypeStr = (String) 
tokenDetails.get(TOKEN_PRINCIPAL_TYPE);
 

usergrid git commit: Fix issue where tokenInfo was not being updated with latest inactive value. Add trace logging to TokenSerializationImpl. Fix issue with push notifications where the InactiveDevice

[mrusso]

Repository: usergrid
Updated Branches:
  refs/heads/token-service-to-dstax 146e47d66 -> bea9a4180


Fix issue where tokenInfo was not being updated with latest inactive value. Add 
trace logging to TokenSerializationImpl. Fix issue with push notifications 
where the InactiveDeviceManager was making a duplicate (yet incorrect) query 
for devices.


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/bea9a418
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/bea9a418
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/bea9a418

Branch: refs/heads/token-service-to-dstax
Commit: bea9a4180ebadb79e7f101cfc6d0d06efc39b359
Parents: 146e47d
Author: Michael Russo 
Authored: Sun Jun 18 12:36:56 2017 -0700
Committer: Michael Russo 
Committed: Sun Jun 18 12:36:56 2017 -0700

--
 .../token/impl/TokenSerializationImpl.java  | 35 ++--
 .../security/tokens/impl/TokenServiceImpl.java  |  9 +++--
 .../notifications/InactiveDeviceManager.java|  6 
 3 files changed, 31 insertions(+), 19 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/usergrid/blob/bea9a418/stack/corepersistence/token/src/main/java/org/apache/usergrid/persistence/token/impl/TokenSerializationImpl.java
--
diff --git 
a/stack/corepersistence/token/src/main/java/org/apache/usergrid/persistence/token/impl/TokenSerializationImpl.java
 
b/stack/corepersistence/token/src/main/java/org/apache/usergrid/persistence/token/impl/TokenSerializationImpl.java
index 2207b99..240da00 100644
--- 
a/stack/corepersistence/token/src/main/java/org/apache/usergrid/persistence/token/impl/TokenSerializationImpl.java
+++ 
b/stack/corepersistence/token/src/main/java/org/apache/usergrid/persistence/token/impl/TokenSerializationImpl.java
@@ -130,22 +130,27 @@ public class TokenSerializationImpl implements 
TokenSerialization {
 public void deleteTokens(final List tokenUUIDs, final ByteBuffer 
principalKeyBuffer){
 
 Preconditions.checkNotNull(tokenUUIDs, "token UUID list is required");
+Preconditions.checkNotNull(tokenUUIDs, "principalKeyBuffer is 
required");
 
-final BatchStatement batchStatement = new BatchStatement();
+logger.trace("deleteTokens, token UUIDs: {}", tokenUUIDs);
 
-tokenUUIDs.forEach( tokenUUID -> batchStatement.add(
-QueryBuilder.delete()
-.from(TOKENS_TABLE)
-.where(QueryBuilder
-.eq("key", DataType.uuid().serialize(tokenUUID, 
ProtocolVersion.NEWEST_SUPPORTED);
+final BatchStatement batchStatement = new BatchStatement();
 
-if(principalKeyBuffer != null){
+tokenUUIDs.forEach( tokenUUID ->
 batchStatement.add(
 QueryBuilder.delete()
-.from(PRINCIPAL_TOKENS_TABLE)
+.from(TOKENS_TABLE)
 .where(QueryBuilder
-.eq("key", principalKeyBuffer)));
-}
+.eq("key", DataType.uuid().serialize(tokenUUID, 
ProtocolVersion.NEWEST_SUPPORTED)))
+)
+);
+
+batchStatement.add(
+QueryBuilder.delete()
+.from(PRINCIPAL_TOKENS_TABLE)
+.where(QueryBuilder
+.eq("key", principalKeyBuffer)));
+
 
 session.execute(batchStatement);
 
@@ -157,6 +162,9 @@ public class TokenSerializationImpl implements 
TokenSerialization {
 
 Preconditions.checkNotNull(tokenUUID, "token UUID is required");
 
+logger.trace("revokeToken, token UUID: {}", tokenUUID);
+
+
 final BatchStatement batchStatement = new BatchStatement();
 
 batchStatement.add(
@@ -188,6 +196,8 @@ public class TokenSerializationImpl implements 
TokenSerialization {
 Preconditions.checkNotNull(inactiveTime, "inactiveTime is required");
 Preconditions.checkNotNull(ttl, "ttl is required");
 
+logger.trace("updateTokenAccessTime, token UUID: {}, accessedTime: {}, 
inactiveTime: {}, ttl: {}",
+tokenUUID, accessedTime, inactiveTime, ttl);
 
 final BatchStatement batchStatement = new BatchStatement();
 final Clause inKey =
@@ -265,6 +275,8 @@ public class TokenSerializationImpl implements 
TokenSerialization {
 
 });
 
+logger.trace("getTokenInfo, info: {}", tokenInfo);
+
 return tokenInfo;
 }
 
@@ -277,6 +289,7 @@ public class TokenSerializationImpl implements 
TokenSerialization {
 Preconditions.checkNotNull(tokenUUID, "tokenInfo is required");
 Preconditions.checkNotNull(ttl, "ttl is required");
 
+logger.trace("putTokenInfo, token UUID: {}, tokenInfo: {}, ttl: {}", 
tokenUUID, tokenInfo, ttl);
 
 final BatchStatement batchStatem

[1/2] usergrid git commit: Finish convert token service persistence to Datastax. Add tests, and rename the package for TokenServiceImpl to impl from cassandra.

[mrusso]

Repository: usergrid
Updated Branches:
  refs/heads/token-service-to-dstax 179f12db0 -> 146e47d66


http://git-wip-us.apache.org/repos/asf/usergrid/blob/146e47d6/stack/services/src/main/java/org/apache/usergrid/security/tokens/cassandra/TokenServiceImpl.java
--
diff --git 
a/stack/services/src/main/java/org/apache/usergrid/security/tokens/cassandra/TokenServiceImpl.java
 
b/stack/services/src/main/java/org/apache/usergrid/security/tokens/cassandra/TokenServiceImpl.java
deleted file mode 100644
index 6ea6de0..000
--- 
a/stack/services/src/main/java/org/apache/usergrid/security/tokens/cassandra/TokenServiceImpl.java
+++ /dev/null
@@ -1,848 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *  http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.usergrid.security.tokens.cassandra;
-
-
-import com.google.inject.Injector;
-import me.prettyprint.hector.api.Keyspace;
-import me.prettyprint.hector.api.beans.HColumn;
-import me.prettyprint.hector.api.mutation.Mutator;
-import org.apache.usergrid.corepersistence.CpEntityManagerFactory;
-import org.apache.usergrid.corepersistence.util.CpNamingUtils;
-import org.apache.usergrid.management.ApplicationCreator;
-import org.apache.usergrid.management.ManagementService;
-import org.apache.usergrid.management.UserInfo;
-import org.apache.usergrid.persistence.EntityManagerFactory;
-import org.apache.usergrid.persistence.cassandra.CassandraService;
-import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
-import org.apache.usergrid.persistence.entities.Application;
-import org.apache.usergrid.security.AuthPrincipalInfo;
-import org.apache.usergrid.security.AuthPrincipalType;
-import org.apache.usergrid.security.sso.SSOProviderFactory;
-import org.apache.usergrid.security.tokens.TokenCategory;
-import org.apache.usergrid.security.tokens.TokenInfo;
-import org.apache.usergrid.security.tokens.TokenService;
-import org.apache.usergrid.security.tokens.exceptions.BadTokenException;
-import org.apache.usergrid.security.tokens.exceptions.ExpiredTokenException;
-import org.apache.usergrid.security.tokens.exceptions.InvalidTokenException;
-import org.apache.usergrid.security.sso.ExternalSSOProvider;
-import org.apache.usergrid.utils.ConversionUtils;
-import org.apache.usergrid.utils.JsonUtils;
-import org.apache.usergrid.utils.UUIDUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.beans.factory.annotation.Qualifier;
-import org.springframework.util.Assert;
-
-import javax.ws.rs.client.Client;
-import java.nio.ByteBuffer;
-import java.util.*;
-
-import static java.lang.System.currentTimeMillis;
-import static me.prettyprint.hector.api.factory.HFactory.createColumn;
-import static me.prettyprint.hector.api.factory.HFactory.createMutator;
-import static org.apache.commons.codec.binary.Base64.decodeBase64;
-import static org.apache.commons.codec.binary.Base64.encodeBase64URLSafeString;
-import static org.apache.commons.codec.digest.DigestUtils.sha;
-import static 
org.apache.usergrid.persistence.cassandra.CassandraPersistenceUtils.getColumnMap;
-import static 
org.apache.usergrid.persistence.cassandra.CassandraService.PRINCIPAL_TOKEN_CF;
-import static 
org.apache.usergrid.persistence.cassandra.CassandraService.TOKENS_CF;
-import static org.apache.usergrid.persistence.cassandra.Serializers.*;
-import static org.apache.usergrid.security.AuthPrincipalType.ADMIN_USER;
-import static org.apache.usergrid.security.tokens.TokenCategory.*;
-import static org.apache.usergrid.utils.ConversionUtils.*;
-import static org.apache.usergrid.utils.MapUtils.hasKeys;
-import static org.apache.usergrid.utils.MapUtils.hashMap;
-import static org.apache.usergrid.utils.UUIDUtils.getTimestampInMillis;
-
-
-public class TokenServiceImpl implements TokenService {
-
-private static final Logger logger = LoggerFactory.getLogger( 
TokenServiceImpl.class );
-
-public static final String PROPERTIES_AUTH_TOKEN_SECRET_SALT = 
"usergrid.auth.token_secret_salt";
-public static final String PROPERTIES_AUTH_TOKEN_EXPIRES_FROM_LAST_USE =
-"usergrid.auth.token_expires_from_last_use";
-pu

[2/2] usergrid git commit: Finish convert token service persistence to Datastax. Add tests, and rename the package for TokenServiceImpl to impl from cassandra.

[mrusso]

Finish convert token service persistence to Datastax. Add tests, and rename the 
package for TokenServiceImpl to impl from cassandra.


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/146e47d6
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/146e47d6
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/146e47d6

Branch: refs/heads/token-service-to-dstax
Commit: 146e47d668788d72c4fc54a61b7155f508b6
Parents: 179f12d
Author: Michael Russo 
Authored: Sat Jun 17 23:47:47 2017 -0700
Committer: Michael Russo 
Committed: Sat Jun 17 23:47:47 2017 -0700

--
 .../src/test/resources/usergrid-test.properties |   2 -
 stack/core/pom.xml  |   6 +
 .../usergrid/corepersistence/CoreModule.java|   2 +
 .../usergrid/corepersistence/CpSetup.java   |  11 -
 .../persistence/cassandra/CassandraService.java |   3 -
 .../map/impl/MapSerializationImpl.java  |  35 +-
 .../persistence/token/TokenSerialization.java   |  50 ++
 .../persistence/token/guice/TokenModule.java|  10 +-
 .../token/impl/TokenSerialization.java  |  50 --
 .../token/impl/TokenSerializationImpl.java  | 482 +--
 .../persistence/token/TestTokenModule.java  |  38 +
 .../token/TokenSerializationTest.java   | 202 +
 .../persistence/token/TokenTestModule.java  |  40 -
 .../resources/usergrid-standalone.properties|   2 -
 .../rest/management/ManagementResource.java |   4 +-
 .../rest/management/users/UserResource.java |   2 +-
 .../rest/management/users/UsersResource.java|   3 +-
 .../rest/management/ManagementResourceIT.java   |   8 +-
 .../security/sso/SSOProviderFactory.java|   2 +-
 .../tokens/cassandra/TokenServiceImpl.java  | 848 ---
 .../security/tokens/impl/TokenServiceImpl.java  | 731 
 .../resources/usergrid-services-context.xml |   3 +-
 .../security/tokens/TokenServiceIT.java |   2 +-
 .../test/resources/testApplicationContext.xml   |   2 +-
 24 files changed, 1253 insertions(+), 1285 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/usergrid/blob/146e47d6/stack/config/src/test/resources/usergrid-test.properties
--
diff --git a/stack/config/src/test/resources/usergrid-test.properties 
b/stack/config/src/test/resources/usergrid-test.properties
index 143d60a..5def24e 100644
--- a/stack/config/src/test/resources/usergrid-test.properties
+++ b/stack/config/src/test/resources/usergrid-test.properties
@@ -104,8 +104,6 @@ collection.uniquevalues.authoritative.region=us-east
 
 usergrid.counter.batch.interval=10
 #usergrid.auth.token_secret_salt=super secret token value
-#usergrid.auth.token_expires_from_last_use=false
-#usergrid.auth.token_refresh_reuses_id=false
 
 # max time to persist tokens for (milliseconds)
 #usergrid.auth.token.persist.expires=0

http://git-wip-us.apache.org/repos/asf/usergrid/blob/146e47d6/stack/core/pom.xml
--
diff --git a/stack/core/pom.xml b/stack/core/pom.xml
index 09047bb..658320e 100644
--- a/stack/core/pom.xml
+++ b/stack/core/pom.xml
@@ -428,6 +428,12 @@
 
 
 org.apache.usergrid
+token
+${project.version}
+
+
+
+org.apache.usergrid
 actorsystem
 ${project.version}
 

http://git-wip-us.apache.org/repos/asf/usergrid/blob/146e47d6/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
--
diff --git 
a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java 
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
index af297f2..ec6b775 100644
--- 
a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
+++ 
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
@@ -44,6 +44,7 @@ import 
org.apache.usergrid.persistence.core.scope.ApplicationScope;
 import org.apache.usergrid.persistence.graph.guice.GraphModule;
 import 
org.apache.usergrid.persistence.graph.serialization.impl.migration.GraphNode;
 import org.apache.usergrid.persistence.index.guice.IndexModule;
+import org.apache.usergrid.persistence.token.guice.TokenModule;
 import org.safehaus.guicyfig.GuicyFigModule;
 
 import java.util.Properties;
@@ -67,6 +68,7 @@ public class CoreModule extends AbstractModule {
 install( new CommonModule());
 install( new LockModule());
 install( new CacheModule());
+install( new TokenModule());
 install( new CollectionModule() {
 /**
  * configure our migration data provider for a

usergrid git commit: Initial commit of new token module to replace Hector for access token storage.

[mrusso]

Repository: usergrid
Updated Branches:
  refs/heads/token-service-to-dstax [created] 179f12db0


Initial commit of new token module to replace Hector for access token storage.


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/179f12db
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/179f12db
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/179f12db

Branch: refs/heads/token-service-to-dstax
Commit: 179f12db0ef55cf9c28936678a933e2de04034eb
Parents: 3f819dc
Author: Michael Russo 
Authored: Fri Jun 16 00:15:14 2017 -0700
Committer: Michael Russo 
Committed: Fri Jun 16 00:15:14 2017 -0700

--
 stack/corepersistence/pom.xml   |   1 +
 stack/corepersistence/token/pom.xml |  70 +++
 .../persistence/token/guice/TokenModule.java|  48 ++
 .../token/impl/TokenSerialization.java  |  50 ++
 .../token/impl/TokenSerializationImpl.java  | 536 +++
 .../persistence/token/TokenTestModule.java  |  40 ++
 .../token/src/test/resources/log4j.properties   |  38 ++
 7 files changed, 783 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/usergrid/blob/179f12db/stack/corepersistence/pom.xml
--
diff --git a/stack/corepersistence/pom.xml b/stack/corepersistence/pom.xml
index 8a45323..45aad4b 100644
--- a/stack/corepersistence/pom.xml
+++ b/stack/corepersistence/pom.xml
@@ -113,6 +113,7 @@ limitations under the License.
 queue
 cache
 actorsystem
+token
 
 
 

http://git-wip-us.apache.org/repos/asf/usergrid/blob/179f12db/stack/corepersistence/token/pom.xml
--
diff --git a/stack/corepersistence/token/pom.xml 
b/stack/corepersistence/token/pom.xml
new file mode 100644
index 000..fcbd73c
--- /dev/null
+++ b/stack/corepersistence/token/pom.xml
@@ -0,0 +1,70 @@
+
+
+http://maven.apache.org/POM/4.0.0";
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 
http://maven.apache.org/xsd/maven-4.0.0.xsd";>
+
+
+persistence
+org.apache.usergrid
+2.2.0-SNAPSHOT
+
+
+4.0.0
+The module for handling access token persistence
+
+token
+Usergrid Token
+
+
+
+
+org.apache.commons
+commons-lang3
+${commons.lang.version}
+
+
+
+org.apache.usergrid
+common
+${project.version}
+
+
+
+
+
+org.apache.usergrid
+common
+${project.version}
+tests
+test
+
+
+
+
+
+
+

http://git-wip-us.apache.org/repos/asf/usergrid/blob/179f12db/stack/corepersistence/token/src/main/java/org/apache/usergrid/persistence/token/guice/TokenModule.java
--
diff --git 
a/stack/corepersistence/token/src/main/java/org/apache/usergrid/persistence/token/guice/TokenModule.java
 
b/stack/corepersistence/token/src/main/java/org/apache/usergrid/persistence/token/guice/TokenModule.java
new file mode 100644
index 000..13d6b43
--- /dev/null
+++ 
b/stack/corepersistence/token/src/main/java/org/apache/usergrid/persistence/token/guice/TokenModule.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ *  contributor license agreements.  The ASF licenses this file to You
+ * under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.  For additional information regarding
+ * copyright in this work, please see the NOTICE file in the top level
+ * directory of this distribution.
+ */
+package org.apache.usergrid.persistence.token.guice;
+
+import com.google.inject.AbstractModule;
+import com.google.inject.Key;
+import com.google.inject.multibindings.Multibinder;
+import org.apache.usergrid.persistence.cache.CacheFactory;
+import org.apache.usergrid.persistence.cache.impl.CacheFactoryImpl;
+import org.apache.usergrid.persistence.cache.impl.TokenSerialization;
+import org.apache.usergrid.persistence.cache.impl.TokenSerializationImpl;
+import org.apache.usergrid.persistence.core.migration.schema.Migration;
+
+
+/**
+

usergrid git commit: Add new parameter that can be used in the JSP viewables.

[mrusso]

Repository: usergrid
Updated Branches:
  refs/heads/hotfix-20160819 ff574d3b3 -> 436cb8046


Add new parameter that can be used in the JSP viewables.


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/436cb804
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/436cb804
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/436cb804

Branch: refs/heads/hotfix-20160819
Commit: 436cb8046d8a23f457a1bb34cfc3d6769da6000d
Parents: ff574d3
Author: Michael Russo 
Authored: Wed May 24 21:45:04 2017 -0700
Committer: Michael Russo 
Committed: Wed May 24 21:45:04 2017 -0700

--
 stack/config/src/main/resources/usergrid-default.properties   | 1 +
 .../apache/usergrid/rest/management/users/UserResource.java   | 7 +++
 2 files changed, 8 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/usergrid/blob/436cb804/stack/config/src/main/resources/usergrid-default.properties
--
diff --git a/stack/config/src/main/resources/usergrid-default.properties 
b/stack/config/src/main/resources/usergrid-default.properties
index d153ef1..6873446 100644
--- a/stack/config/src/main/resources/usergrid-default.properties
+++ b/stack/config/src/main/resources/usergrid-default.properties
@@ -750,6 +750,7 @@ usergrid.api.url.base=http://localhost:8080/ROOT
 # instead, use character class ([.] instead of backslash-period)
 usergrid.org.config.property.regex=usergrid[.]view[.].*
 
+usergrid.viewable.loginEndpoint=http://localhost:8080
 
 
 ###  Usergrid Email Templates  

http://git-wip-us.apache.org/repos/asf/usergrid/blob/436cb804/stack/rest/src/main/java/org/apache/usergrid/rest/management/users/UserResource.java
--
diff --git 
a/stack/rest/src/main/java/org/apache/usergrid/rest/management/users/UserResource.java
 
b/stack/rest/src/main/java/org/apache/usergrid/rest/management/users/UserResource.java
index 066f734..95f607b 100644
--- 
a/stack/rest/src/main/java/org/apache/usergrid/rest/management/users/UserResource.java
+++ 
b/stack/rest/src/main/java/org/apache/usergrid/rest/management/users/UserResource.java
@@ -66,6 +66,9 @@ public class UserResource extends AbstractContextResource {
 
 String token = null;
 
+String loginEndpoint;
+
+
 
 public UserResource() {
 }
@@ -296,6 +299,7 @@ public class UserResource extends AbstractContextResource {
 if ( ( password1 != null ) && password1.equals( password2 
) ) {
 management.setAdminUserPassword( user.getUuid(), 
password1 );
 management.revokeAccessTokenForAdminUser( 
user.getUuid(), token );
+loginEndpoint = 
properties.getProperty("usergrid.viewable.loginEndpoint");
 return handleViewable( "resetpw_set_success", this, 
organizationId );
 }
 else {
@@ -342,6 +346,9 @@ public class UserResource extends AbstractContextResource {
 return errorMsg;
 }
 
+public String getLoginEndpoint() {
+return loginEndpoint;
+}
 
 public String getToken() {
 return token;

usergrid git commit: Allow name and any property set with aliasProperty to be changed.

[mrusso]

Repository: usergrid
Updated Branches:
  refs/heads/master bafd46274 -> 3f819dc06


Allow name and any property set with aliasProperty to be changed.


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/3f819dc0
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/3f819dc0
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/3f819dc0

Branch: refs/heads/master
Commit: 3f819dc0679f84edb57c52e69b58622417cfd59f
Parents: bafd462
Author: Michael Russo 
Authored: Wed May 17 13:50:10 2017 -0700
Committer: Michael Russo 
Committed: Wed May 17 13:50:10 2017 -0700

--
 .../src/main/java/org/apache/usergrid/persistence/Schema.java | 2 +-
 .../java/org/apache/usergrid/persistence/schema/PropertyInfo.java | 3 ---
 2 files changed, 1 insertion(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/usergrid/blob/3f819dc0/stack/core/src/main/java/org/apache/usergrid/persistence/Schema.java
--
diff --git 
a/stack/core/src/main/java/org/apache/usergrid/persistence/Schema.java 
b/stack/core/src/main/java/org/apache/usergrid/persistence/Schema.java
index 6b8f139..87a9a51 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/Schema.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/Schema.java
@@ -1252,7 +1252,7 @@ public class Schema implements SchemaManager {
 property.setName( PROPERTY_NAME );
 property.setRequired( false );
 property.setType( String.class );
-property.setMutable( false );
+property.setMutable( true );
 property.setAliasProperty( true );
 property.setIndexed( true );
 property.setBasic( true );

http://git-wip-us.apache.org/repos/asf/usergrid/blob/3f819dc0/stack/core/src/main/java/org/apache/usergrid/persistence/schema/PropertyInfo.java
--
diff --git 
a/stack/core/src/main/java/org/apache/usergrid/persistence/schema/PropertyInfo.java
 
b/stack/core/src/main/java/org/apache/usergrid/persistence/schema/PropertyInfo.java
index c5d2d9c..14d5e30 100644
--- 
a/stack/core/src/main/java/org/apache/usergrid/persistence/schema/PropertyInfo.java
+++ 
b/stack/core/src/main/java/org/apache/usergrid/persistence/schema/PropertyInfo.java
@@ -142,9 +142,6 @@ public class PropertyInfo {
 
 public void setAliasProperty( boolean aliasProperty ) {
 this.aliasProperty = aliasProperty;
-if ( aliasProperty ) {
-mutable = false;
-}
 }
 
 

[1/2] usergrid git commit: Add a new Asset binary provider for Google Cloud Storage. Refactor the BinaryStore implementation to follow a Factory pattern.

[mrusso]

Repository: usergrid
Updated Branches:
  refs/heads/master 949b46506 -> bafd46274


http://git-wip-us.apache.org/repos/asf/usergrid/blob/bafd4627/stack/services/src/main/java/org/apache/usergrid/services/assets/data/GoogleBinaryStore.java
--
diff --git 
a/stack/services/src/main/java/org/apache/usergrid/services/assets/data/GoogleBinaryStore.java
 
b/stack/services/src/main/java/org/apache/usergrid/services/assets/data/GoogleBinaryStore.java
new file mode 100644
index 000..ae661f1
--- /dev/null
+++ 
b/stack/services/src/main/java/org/apache/usergrid/services/assets/data/GoogleBinaryStore.java
@@ -0,0 +1,223 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.usergrid.services.assets.data;
+
+
+import com.google.api.services.storage.StorageScopes;
+import com.google.auth.oauth2.GoogleCredentials;
+import com.google.cloud.HttpTransportOptions;
+import com.google.cloud.TransportOptions;
+import com.google.cloud.WriteChannel;
+import com.google.cloud.storage.*;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.lang.RandomStringUtils;
+import org.apache.usergrid.persistence.Entity;
+import org.apache.usergrid.persistence.EntityManager;
+import org.apache.usergrid.persistence.EntityManagerFactory;
+import org.apache.usergrid.utils.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.*;
+import java.nio.ByteBuffer;
+import java.security.GeneralSecurityException;
+import java.util.Map;
+import java.util.Properties;
+import java.util.UUID;
+import java.util.concurrent.atomic.AtomicLong;
+
+
+public class GoogleBinaryStore implements BinaryStore {
+
+private static final Logger logger = 
LoggerFactory.getLogger(GoogleBinaryStore.class);
+private static final long FIVE_MB = ( FileUtils.ONE_MB * 5 );
+
+
+private EntityManagerFactory entityManagerFactory;
+
+private Properties properties;
+
+private String bucketName;
+
+private Storage instance = null;
+
+private String reposLocation = FileUtils.getTempDirectoryPath();
+
+public GoogleBinaryStore(Properties properties,
+ EntityManagerFactory entityManagerFactory,
+ String reposLocation) throws IOException, 
GeneralSecurityException {
+this.entityManagerFactory = entityManagerFactory;
+this.properties = properties;
+this.reposLocation = reposLocation;
+}
+
+private synchronized Storage getService() throws IOException, 
GeneralSecurityException {
+
+logger.trace("Getting Google Cloud Storage service");
+
+// leave this here for tests because they do things like manipulate 
properties dynamically to test invalid values
+this.bucketName = properties.getProperty( "usergrid.binary.bucketname" 
);
+
+
+if (instance == null) {
+
+// Google provides different authentication types which are 
different based on if the application is
+// running within GCE(Google Compute Engine) or GAE (Google App 
Engine). If Usergrid is running in
+// GCE or GAE, the SDK will automatically authenticate and get 
access to
+// cloud storage. Else, the full path to a credential file should 
be provided in the following environment variable
+//
+// GOOGLE_APPLICATION_CREDENTIALS
+//
+// The SDK will attempt to load the credential file for a service 
account. See the following
+// for more info: 
https://developers.google.com/identity/protocols/application-default-credentials#howtheywork
+GoogleCredentials credentials = 
GoogleCredentials.getApplicationDefault().createScoped(StorageScopes.all());
+
+
+final TransportOptions transportOptions = 
HttpTransportOptions.newBuilder()
+.setConnectTimeout(3) // in milliseconds
+.setReadTimeout(3) // in milliseconds
+.build();
+
+instance = StorageOptions.newBuilder()
+.setCredentials(credentials)
+.setTransportOptions(transportOptions)
+.build()
+ 

[2/2] usergrid git commit: Add a new Asset binary provider for Google Cloud Storage. Refactor the BinaryStore implementation to follow a Factory pattern.

[mrusso]

Add a new Asset binary provider for Google Cloud Storage. Refactor the 
BinaryStore implementation to follow a Factory pattern.


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/bafd4627
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/bafd4627
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/bafd4627

Branch: refs/heads/master
Commit: bafd462744803f28175b13d0c205e3b029a6ac54
Parents: 949b465
Author: Michael Russo 
Authored: Wed Apr 26 15:43:40 2017 -0700
Committer: Michael Russo 
Committed: Wed Apr 26 15:43:40 2017 -0700

--
 .../main/resources/usergrid-default.properties  |  22 +-
 .../test/resources/usergrid-test-context.xml|   6 +-
 .../rest/applications/ServiceResource.java  |  62 +--
 .../applications/assets/AssetsResource.java |  38 +-
 .../main/resources/usergrid-rest-context.xml|  15 +-
 .../applications/assets/AssetResourceIT.java|   2 +-
 .../applications/assets/AwsAssetResourceIT.java |   4 +-
 .../assets/GoogleAssetResourceIT.java   | 419 +++
 .../applications/assets/aws/NoAWSCredsRule.java | 124 --
 .../assets/rules/NoAWSCredsRule.java| 136 ++
 .../assets/rules/NoGoogleCredsRule.java | 118 ++
 stack/services/pom.xml  |   6 +
 .../services/assets/BinaryStoreFactory.java |  83 
 .../services/assets/data/AWSBinaryStore.java| 320 ++
 .../assets/data/AwsSdkS3BinaryStore.java| 317 --
 .../services/assets/data/GoogleBinaryStore.java | 223 ++
 .../assets/data/LocalFileBinaryStore.java   |  17 +-
 .../services/assets/data/S3BinaryStore.java | 346 ---
 .../test/resources/usergrid-test-context.xml|  12 +-
 19 files changed, 1381 insertions(+), 889 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/usergrid/blob/bafd4627/stack/config/src/main/resources/usergrid-default.properties
--
diff --git a/stack/config/src/main/resources/usergrid-default.properties 
b/stack/config/src/main/resources/usergrid-default.properties
index 9c75ef0..0d1a193 100644
--- a/stack/config/src/main/resources/usergrid-default.properties
+++ b/stack/config/src/main/resources/usergrid-default.properties
@@ -565,13 +565,29 @@ usergrid.binary.bucketname=usergrid-binaries
 #
 usergrid.temp.files=/tmp/usergrid
 
-# Set the implementation of binary uploading to be used
-# Aws for aws s3 uploading, local
+# Set the implementation of binary uploading to be used. Valid values:
+#
+# local - uses the local file system and "usergrid.temp.files" specifies the 
location
+# aws - uses AWS Simple Storage Service (S3)
+# google - uses Google Cloud Platform Storage service (GCS)
+#
+#
+# If Google is specified, the following ENVIRONMENT variable must be set on 
the system running Usergrid:
+#
+# GOOGLE_APPLICATION_CREDENTIALS=/full/path/to/credentialfile.json
+#
+# See: 
https://developers.google.com/identity/protocols/application-default-credentials#howtheywork
+#
+#
+# If AWS is specified, the following JVM Arguments must be set:
+#
+# AWS_ACCESS_KEY_ID=awsKeyId
+# AWS_SECRET_KEY=awsKeySecret
+#
 usergrid.binary.uploader=local
 
 
 
-
 ###  Usergrid Admin  ##
 #
 # Usergrid has a sysadmin user which has access to the complete Usergrid system

http://git-wip-us.apache.org/repos/asf/usergrid/blob/bafd4627/stack/query-validator/src/test/resources/usergrid-test-context.xml
--
diff --git a/stack/query-validator/src/test/resources/usergrid-test-context.xml 
b/stack/query-validator/src/test/resources/usergrid-test-context.xml
index 7affe6d..19af73d 100644
--- a/stack/query-validator/src/test/resources/usergrid-test-context.xml
+++ b/stack/query-validator/src/test/resources/usergrid-test-context.xml
@@ -44,7 +44,11 @@
 
 
 
-
+
+
+
+
+
 
 
 

http://git-wip-us.apache.org/repos/asf/usergrid/blob/bafd4627/stack/rest/src/main/java/org/apache/usergrid/rest/applications/ServiceResource.java
--
diff --git 
a/stack/rest/src/main/java/org/apache/usergrid/rest/applications/ServiceResource.java
 
b/stack/rest/src/main/java/org/apache/usergrid/rest/applications/ServiceResource.java
index 3835b75..9373f5e 100644
--- 
a/stack/rest/src/main/java/org/apache/usergrid/rest/applications/ServiceResource.java
+++ 
b/stack/rest/src/main/java/org/apache/usergrid/rest/applications/ServiceResource.java
@@ -20,6 +20,7 @@ package org.apache.usergrid.rest.applications;
 import com.amazonaws.AmazonServiceException;
 import com.fasterxml.jackson.databind.ObjectMapper;
 

usergrid git commit: Add new push notification test and remove duplicate index processing on entity create/update.

[mrusso]

Repository: usergrid
Updated Branches:
  refs/heads/master d3e988bcb -> 949b46506


Add new push notification test and remove duplicate index processing on entity 
create/update.


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/949b4650
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/949b4650
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/949b4650

Branch: refs/heads/master
Commit: 949b4650608ce9ecd3d3bea10ff81d7fe7f8be2f
Parents: d3e988b
Author: Michael Russo 
Authored: Wed Apr 5 19:01:11 2017 -0700
Committer: Michael Russo 
Committed: Wed Apr 5 19:01:11 2017 -0700

--
 .../batch/service/JobSchedulerService.java  |  8 ++--
 .../batch/service/SchedulerServiceImpl.java |  2 +-
 .../asyncevents/AsyncEventServiceImpl.java  | 14 ++
 .../org/apache/usergrid/ServiceITSetup.java |  3 ++
 .../org/apache/usergrid/ServiceITSetupImpl.java | 17 ---
 .../apns/NotificationsServiceIT.java| 47 +---
 6 files changed, 62 insertions(+), 29 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/usergrid/blob/949b4650/stack/core/src/main/java/org/apache/usergrid/batch/service/JobSchedulerService.java
--
diff --git 
a/stack/core/src/main/java/org/apache/usergrid/batch/service/JobSchedulerService.java
 
b/stack/core/src/main/java/org/apache/usergrid/batch/service/JobSchedulerService.java
index 807daa3..23a86af 100644
--- 
a/stack/core/src/main/java/org/apache/usergrid/batch/service/JobSchedulerService.java
+++ 
b/stack/core/src/main/java/org/apache/usergrid/batch/service/JobSchedulerService.java
@@ -129,9 +129,9 @@ public class JobSchedulerService extends 
AbstractScheduledService {
 }
 
 for ( JobDescriptor jd : activeJobs ) {
-logger.info( "Submitting work for {}", jd );
+logger.debug( "Submitting work for {}", jd );
 submitWork( jd );
-logger.info( "Work submitted for {}", jd );
+logger.debug( "Work submitted for {}", jd );
 }
 }
 }
@@ -228,7 +228,7 @@ public class JobSchedulerService extends 
AbstractScheduledService {
 // TODO wrap and throw specifically typed exception for 
onFailure,
 // needs jobId
 
-logger.info( "Starting job {} with execution data {}", job, 
execution );
+logger.debug( "Starting job {} with execution data {}", job, 
execution );
 
 job.execute( execution );
 
@@ -259,7 +259,7 @@ public class JobSchedulerService extends 
AbstractScheduledService {
 
 //TODO, refactor into the execution itself for checking if done
 if ( execution.getStatus() == Status.IN_PROGRESS ) {
-logger.info( "Successful completion of bulkJob {}", 
execution );
+logger.debug( "Successful completion of bulkJob {}", 
execution );
 execution.completed();
 }
 

http://git-wip-us.apache.org/repos/asf/usergrid/blob/949b4650/stack/core/src/main/java/org/apache/usergrid/batch/service/SchedulerServiceImpl.java
--
diff --git 
a/stack/core/src/main/java/org/apache/usergrid/batch/service/SchedulerServiceImpl.java
 
b/stack/core/src/main/java/org/apache/usergrid/batch/service/SchedulerServiceImpl.java
index af933f6..9b3714c 100644
--- 
a/stack/core/src/main/java/org/apache/usergrid/batch/service/SchedulerServiceImpl.java
+++ 
b/stack/core/src/main/java/org/apache/usergrid/batch/service/SchedulerServiceImpl.java
@@ -305,7 +305,7 @@ public class SchedulerServiceImpl implements 
SchedulerService, JobAccessor, JobR
 getEm().update( data );
 }
 
-logger.info( "Updating stats for job {}", data.getJobName() );
+logger.debug( "Updating stats for job {}", data.getJobName() );
 
 getEm().update( stat );
 }

http://git-wip-us.apache.org/repos/asf/usergrid/blob/949b4650/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventServiceImpl.java
--
diff --git 
a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventServiceImpl.java
 
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventServiceImpl.java
index e565754..75d2ce0 100644
--- 
a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventServiceImpl.java
+++ 
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventServiceImpl.java
@@ -420,8 +420,8 @@ public 

[2/3] usergrid git commit: Switch DISTRIBUTED database queueing to default not cache in memory as the in memory impl causes duplicate messgae processing quite often at the moment.

[mrusso]

http://git-wip-us.apache.org/repos/asf/usergrid/blob/d3e988bc/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/qakka/distributed/impl/DistributedQueueServiceImpl.java
--
diff --git 
a/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/qakka/distributed/impl/DistributedQueueServiceImpl.java
 
b/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/qakka/distributed/impl/DistributedQueueServiceImpl.java
index 98e055a..b1f72aa 100644
--- 
a/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/qakka/distributed/impl/DistributedQueueServiceImpl.java
+++ 
b/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/qakka/distributed/impl/DistributedQueueServiceImpl.java
@@ -20,6 +20,8 @@
 package org.apache.usergrid.persistence.qakka.distributed.impl;
 
 import akka.actor.ActorRef;
+import akka.actor.ActorSystem;
+import akka.dispatch.OnFailure;
 import akka.pattern.Patterns;
 import akka.util.Timeout;
 import com.codahale.metrics.*;
@@ -42,7 +44,9 @@ import 
org.apache.usergrid.persistence.qakka.serialization.queuemessages.Databas
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import scala.concurrent.Await;
+import scala.concurrent.ExecutionContext;
 import scala.concurrent.Future;
+import scala.concurrent.Promise;
 
 import java.lang.reflect.Method;
 import java.util.*;
@@ -235,20 +239,21 @@ public class DistributedQueueServiceImpl implements 
DistributedQueueService {
 public Collection getNextMessagesInternal( String 
queueName, int count ) {
 
 if ( actorSystemManager.getClientActor() == null || 
!actorSystemManager.isReady() ) {
-logger.error("Akka Actor System is not ready yet for requests.");
-return Collections.EMPTY_LIST;
+logger.warn("Akka Actor System is not ready yet for requests.");
+return Collections.emptyList();
 }
 
 int maxRetries = qakkaFig.getMaxGetRetries();
 int tries = 0;
 
+boolean interrupted = false;
+
 QueueGetRequest request = new QueueGetRequest( queueName, count );
 while ( ++tries < maxRetries ) {
 try {
 Timeout t = new Timeout( qakkaFig.getGetTimeoutSeconds(), 
TimeUnit.SECONDS );
 
 // ask ClientActor and wait (up to timeout) for response
-
 Future fut = Patterns.ask( 
actorSystemManager.getClientActor(), request, t );
 Object responseObject = Await.result( fut, t.duration() );
 
@@ -259,8 +264,8 @@ public class DistributedQueueServiceImpl implements 
DistributedQueueService {
 if ( response != null && response instanceof 
QueueGetResponse) {
 QueueGetResponse qprm = (QueueGetResponse)response;
 if ( qprm.isSuccess() ) {
-if (tries > 1) {
-logger.warn( "getNextMessage {} SUCCESS after 
{} tries", queueName, tries );
+if (tries > 1 && !interrupted) {
+logger.warn( "getNextMessage for queue {} 
SUCCESS after {} tries", queueName, tries );
 }
 }
 return qprm.getQueueMessages();
@@ -284,10 +289,13 @@ public class DistributedQueueServiceImpl implements 
DistributedQueueService {
 }
 
 } catch ( TimeoutException e ) {
-logger.trace("TIMEOUT popping to queue " + queueName + " 
retrying " + tries, e );
-
-} catch ( Exception e ) {
-logger.debug("ERROR popping to queue " + queueName + " 
retrying " + tries, e );
+logger.warn("TIMEOUT popping queue " + queueName + ", attempt: 
 " + tries, e );
+} catch(InterruptedException e){
+interrupted = true;
+// this might happen, retry the ask again
+logger.trace("Thread was marked interrupted so unable to wait 
for the result, attempt: {}", tries);
+}catch ( Exception e ) {
+logger.error("ERROR popping queue " + queueName + ", attempt: 
" + tries, e );
 }
 }
 

http://git-wip-us.apache.org/repos/asf/usergrid/blob/d3e988bc/stack/corepersistence/queue/src/test/java/org/apache/usergrid/persistence/qakka/distributed/actors/ShardAllocatorTest.java
--
diff --git 
a/stack/corepersistence/queue/src/test/java/org/apache/usergrid/persistence/qakka/distributed/actors/ShardAllocatorTest.java
 
b/stack/corepersistence/queue/src/test/java/org/apache/usergrid/persistence/qakka/distributed/actors/ShardAllocatorTest.java
index 11f3d08..4745cb1 100644
--- 
a/stack/corepersistence/queue/src/test/java/org/apache/usergrid/persistence/qakka/distributed/actors/ShardAllocatorTest.java
+++ 

[3/3] usergrid git commit: Switch DISTRIBUTED database queueing to default not cache in memory as the in memory impl causes duplicate messgae processing quite often at the moment.

[mrusso]

Switch DISTRIBUTED database queueing to default not cache in memory as the in 
memory impl causes duplicate messgae processing quite often at the moment.

 - includes making all the tests work without in-memory queue fronting the 
database queue which really means adding some more delay in tests
 - the tests now are actually faster now because the original refreshIndex() 
created and queried random entities which took longer in most cases
 - uncommented the checkReceipts function in Notification tests as these are 
now passing, with an added fix for duplicate receipt creation
 - some logging updates in the distributed database queueing impl (Qakka)
 - increased the default take to 500 from the queue when DISTRIBUTED database 
queueing is configured ( which is the default now )
 - Notifications Queue Listener thread names have a random identifier in 
included
 - reduced the DISTRIBUTED database queueing default long poll to 1 second from 
5 seconds


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/d3e988bc
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/d3e988bc
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/d3e988bc

Branch: refs/heads/master
Commit: d3e988bcbb7eb417c84cfda7396ec3506521aa37
Parents: 8b63aae
Author: Michael Russo 
Authored: Sun Apr 2 16:14:05 2017 -0700
Committer: Michael Russo 
Committed: Sun Apr 2 16:14:05 2017 -0700

--
 .../corepersistence/CpEntityManager.java| 44 +-
 .../asyncevents/AsyncIndexProvider.java |  5 ++
 .../java/org/apache/usergrid/Application.java   |  4 +-
 .../org/apache/usergrid/CoreApplication.java| 22 ++---
 .../org/apache/usergrid/CoreITSetupImpl.java| 20 -
 .../org/apache/usergrid/TestEntityIndex.java|  1 +
 .../corepersistence/AggregationServiceTest.java | 15 ++--
 .../corepersistence/StaleIndexCleanupTest.java  | 17 ++--
 .../persistence/ApplicationServiceIT.java   |  6 +-
 .../usergrid/persistence/CollectionIT.java  | 74 
 .../usergrid/persistence/CountingMutatorIT.java |  4 +-
 .../persistence/EntityConnectionsIT.java| 16 ++--
 .../usergrid/persistence/EntityManagerIT.java   | 20 ++---
 .../org/apache/usergrid/persistence/GeoIT.java  | 32 +++
 .../persistence/GeoQueryBooleanTest.java|  4 +-
 .../apache/usergrid/persistence/IndexIT.java| 18 ++--
 .../usergrid/persistence/PathQueryIT.java   |  7 +-
 .../usergrid/persistence/PermissionsIT.java |  6 +-
 .../usergrid/persistence/RebuildIndexTest.java  | 26 +++---
 .../cassandra/EntityManagerFactoryImplIT.java   | 13 +--
 .../persistence/query/ConnectionHelper.java |  2 +-
 .../query/IntersectionTransitivePagingIT.java   |  2 +-
 .../query/IntersectionUnionPagingIT.java|  2 +-
 .../persistence/query/IteratingQueryIT.java | 32 +++
 .../persistence/query/NotSubPropertyIT.java |  2 +-
 .../persistence/query/ParenthesisProblemIT.java |  2 +-
 .../resources/usergrid-custom-test.properties   |  3 +
 .../usergrid/persistence/qakka/QakkaFig.java|  4 +-
 .../qakka/core/impl/InMemoryQueue.java  |  4 +-
 .../core/impl/QueueMessageManagerImpl.java  |  9 +-
 .../distributed/actors/QueueActorHelper.java| 15 +++-
 .../distributed/actors/QueueActorRouter.java|  2 +-
 .../distributed/actors/ShardAllocator.java  |  4 +-
 .../impl/DistributedQueueServiceImpl.java   | 26 --
 .../distributed/actors/ShardAllocatorTest.java  |  3 +
 .../queue/src/test/resources/qakka.properties   |  5 +-
 .../usergrid/rest/CollectionMetadataIT.java |  4 +-
 .../apache/usergrid/rest/NotificationsIT.java   | 14 ++-
 .../apache/usergrid/rest/PartialUpdateTest.java |  6 +-
 .../apache/usergrid/rest/SystemResourceIT.java  |  3 +-
 .../rest/applications/ApplicationCreateIT.java  |  2 +-
 .../rest/applications/ApplicationDeleteIT.java  |  6 +-
 .../applications/ApplicationResourceIT.java | 12 +--
 .../applications/assets/AssetResourceIT.java| 26 +++---
 .../applications/assets/AwsAssetResourceIT.java | 22 ++---
 .../collection/BrowserCompatibilityTest.java|  2 +-
 .../collection/CollectionsResourceIT.java   | 70 +++
 .../collection/DuplicateNameIT.java |  2 +-
 .../activities/ActivityResourceIT.java  |  8 +-
 .../collection/activities/PutTest.java  |  6 +-
 .../collection/devices/DevicesResourceIT.java   | 12 +--
 .../collection/groups/GroupResourceIT.java  | 39 +
 .../collection/paging/PagingResourceIT.java | 10 +--
 .../users/ConnectionResourceTest.java   | 22 ++---
 .../collection/users/OwnershipResourceIT.java   | 24 +++---
 .../collection/users/PermissionsResourceIT.java | 58 ++---
 .../collection/users/RetrieveUsersTest.java |  8 +-
 .../collection/users/UserResourceIT.java| 89 ++--
 .../applications/events/EventsResourceI

[1/3] usergrid git commit: Switch DISTRIBUTED database queueing to default not cache in memory as the in memory impl causes duplicate messgae processing quite often at the moment.

[mrusso]

Repository: usergrid
Updated Branches:
  refs/heads/master 8b63aae7d -> d3e988bcb


http://git-wip-us.apache.org/repos/asf/usergrid/blob/d3e988bc/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/users/UserResourceIT.java
--
diff --git 
a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/users/UserResourceIT.java
 
b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/users/UserResourceIT.java
index f8cb9d4e..af87ca5 100644
--- 
a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/users/UserResourceIT.java
+++ 
b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/users/UserResourceIT.java
@@ -75,7 +75,7 @@ public class UserResourceIT extends AbstractRestIT {
 usersResource = this.app().collection("users");
 userResource = this.app().collection("user");
 
-clientSetup.refreshIndex();
+waitForQueueDrainAndRefreshIndex();
 }
 
 @Test
@@ -249,14 +249,14 @@ public class UserResourceIT extends AbstractRestIT {
 // same as above, but with actor partially filled out
 
 Entity entity = 
usersResource.entity(userId.toString()).activities().post(activity);
-refreshIndex();
+waitForQueueDrainAndRefreshIndex();
 
 UUID firstActivityId = entity.getUuid();
 
 activity = new ActivityEntity("r...@rodsimpson.com", "POST", "activity 
2");
 entity = 
usersResource.entity(userId.toString()).activities().post(activity);
 
-refreshIndex();
+waitForQueueDrainAndRefreshIndex();
 
 UUID secondActivityId = entity.getUuid();
 
@@ -284,7 +284,7 @@ public class UserResourceIT extends AbstractRestIT {
 map.put("email", email);
 
 Entity userEntity = usersResource.post(new Entity(map));
-refreshIndex();
+waitForQueueDrainAndRefreshIndex();
 
 // get the user with username property that has an email value
 Entity testUser = usersResource.entity(username).get();
@@ -315,7 +315,7 @@ public class UserResourceIT extends AbstractRestIT {
 map.put("email", email);
 
 usersResource.post(new Entity(map));
-refreshIndex();
+waitForQueueDrainAndRefreshIndex();
 
 // get the user with email property value
 // get the user with username property that has an email value
@@ -339,7 +339,7 @@ public class UserResourceIT extends AbstractRestIT {
 map.put("email", email);
 
 usersResource.post(new Entity(map));
-refreshIndex();
+waitForQueueDrainAndRefreshIndex();
 
 // get the user with email property value
 // get the user with username property that has an email value
@@ -409,7 +409,7 @@ public class UserResourceIT extends AbstractRestIT {
 Entity entity = usersResource.post(user);
 UUID createdId = entity.getUuid();
 
-refreshIndex();
+waitForQueueDrainAndRefreshIndex();
 Collection results = usersResource.get(new 
QueryParameters().setQuery(String.format("name = '%s'", name)));
 entity = new User(results.getResponse().getEntities().get(0));
 assertEquals(createdId, entity.getUuid());
@@ -429,13 +429,13 @@ public class UserResourceIT extends AbstractRestIT {
 
 UUID createdId = entity.getUuid();
 
-refreshIndex();
+waitForQueueDrainAndRefreshIndex();
 
 Entity newEntity = usersResource.entity(createdId.toString()).get();
 
 userResource.entity(newEntity).delete();
 
-refreshIndex();
+waitForQueueDrainAndRefreshIndex();
 
 Collection results = usersResource.get(
 new QueryParameters().setQuery(String.format("username = '%s'", 
username)));
@@ -460,7 +460,7 @@ public class UserResourceIT extends AbstractRestIT {
 User entity = new User(username, name, email, "password");
 
 entity = new User(usersResource.post(entity));
-refreshIndex();
+waitForQueueDrainAndRefreshIndex();
 
 UUID firstCreatedId = entity.getUuid();
 username = "username2";
@@ -470,7 +470,7 @@ public class UserResourceIT extends AbstractRestIT {
 entity = new User(username, name, email, "password");
 
 entity = new User(usersResource.post(entity));
-refreshIndex();
+waitForQueueDrainAndRefreshIndex();
 
 UUID secondCreatedId = entity.getUuid();
 
@@ -484,7 +484,7 @@ public class UserResourceIT extends AbstractRestIT {
 
 assertEquals(secondCreatedId.toString(), conn1.getUuid().toString());
 
-refreshIndex();
+waitForQueueDrainAndRefreshIndex();
 
 
 Entity conn2 = usersResource.entity(
@@ -492,7 +492,7 @@ public class UserResourceIT extends AbstractRestIT {
 
 assertEquals(secondCreatedId.toString(), conn2.getUuid().toString());
 
-refreshIndex();
+waitForQueueDrainAndRefreshInd

[1/4] usergrid git commit: Initial commit of export application API.

[mrusso]

Repository: usergrid
Updated Branches:
  refs/heads/master ea1ba360d -> 8b63aae7d


Initial commit of export application API.


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/82e7ec57
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/82e7ec57
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/82e7ec57

Branch: refs/heads/master
Commit: 82e7ec57bef24c8b75d3f3479952522fdd916bfa
Parents: ea1ba36
Author: Michael Russo 
Authored: Mon Mar 27 00:01:38 2017 -0700
Committer: Michael Russo 
Committed: Mon Mar 27 00:01:38 2017 -0700

--
 .../usergrid/corepersistence/CoreModule.java|   4 +
 .../export/ExportRequestBuilder.java|  47 
 .../export/ExportRequestBuilderImpl.java|  65 +
 .../corepersistence/export/ExportService.java   |  49 
 .../export/ExportServiceImpl.java   | 235 +++
 .../util/ObjectJsonSerializer.java  |  17 ++
 .../rest/applications/ApplicationResource.java  |  48 
 7 files changed, 465 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/usergrid/blob/82e7ec57/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
--
diff --git 
a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java 
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
index ef4bb04..af297f2 100644
--- 
a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
+++ 
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
@@ -20,6 +20,8 @@ import com.google.inject.*;
 import com.google.inject.assistedinject.FactoryModuleBuilder;
 import com.google.inject.multibindings.Multibinder;
 import org.apache.usergrid.corepersistence.asyncevents.*;
+import org.apache.usergrid.corepersistence.export.ExportService;
+import org.apache.usergrid.corepersistence.export.ExportServiceImpl;
 import org.apache.usergrid.corepersistence.index.*;
 import org.apache.usergrid.corepersistence.migration.CoreMigration;
 import org.apache.usergrid.corepersistence.migration.CoreMigrationPlugin;
@@ -137,6 +139,8 @@ public class CoreModule extends AbstractModule {
 
 bind( ReIndexService.class ).to( ReIndexServiceImpl.class );
 
+bind( ExportService.class ).to( ExportServiceImpl.class );
+
 install( new FactoryModuleBuilder().implement( 
AggregationService.class, AggregationServiceImpl.class )
.build( 
AggregationServiceFactory.class ) );
 

http://git-wip-us.apache.org/repos/asf/usergrid/blob/82e7ec57/stack/core/src/main/java/org/apache/usergrid/corepersistence/export/ExportRequestBuilder.java
--
diff --git 
a/stack/core/src/main/java/org/apache/usergrid/corepersistence/export/ExportRequestBuilder.java
 
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/export/ExportRequestBuilder.java
new file mode 100644
index 000..71c64f6
--- /dev/null
+++ 
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/export/ExportRequestBuilder.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.export;
+
+
+import com.google.common.base.Optional;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+
+
+/**
+ * A builder interface to build our re-index request
+ */
+public interface ExportRequestBuilder {
+
+/**
+ * Set the application id
+ */
+ExportRequestBuilder withApplicationId(final UUID applicationId);
+
+
+/**
+ * Get the application scope
+ * @return
+ */
+Optional getApplicationScope();
+
+}

http://git-wip-us.apache.org/repos/asf/usergrid/blob/82e7ec57/stack/core/src/main/java/org/apache/usergrid/corepersistence/export/ExportRequestBuilderImpl.java
--

[2/4] usergrid git commit: Batch write to the export stream and use a cleaner filename.

[mrusso]

Batch write to the export stream and use a cleaner filename.


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/2d56d813
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/2d56d813
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/2d56d813

Branch: refs/heads/master
Commit: 2d56d813a9d99295ef72ab055e8600f62df364e9
Parents: 82e7ec5
Author: Michael Russo 
Authored: Mon Mar 27 18:48:49 2017 -0700
Committer: Michael Russo 
Committed: Mon Mar 27 18:48:49 2017 -0700

--
 .../export/ExportServiceImpl.java   | 153 ---
 .../rest/applications/ApplicationResource.java  |   4 +-
 2 files changed, 104 insertions(+), 53 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/usergrid/blob/2d56d813/stack/core/src/main/java/org/apache/usergrid/corepersistence/export/ExportServiceImpl.java
--
diff --git 
a/stack/core/src/main/java/org/apache/usergrid/corepersistence/export/ExportServiceImpl.java
 
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/export/ExportServiceImpl.java
index ebbcc58..47d6982 100644
--- 
a/stack/core/src/main/java/org/apache/usergrid/corepersistence/export/ExportServiceImpl.java
+++ 
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/export/ExportServiceImpl.java
@@ -48,7 +48,9 @@ import rx.schedulers.Schedulers;
 
 
 import java.io.*;
+import java.util.HashMap;
 import java.util.Map;
+import java.util.concurrent.atomic.AtomicInteger;
 import java.util.zip.ZipEntry;
 import java.util.zip.ZipOutputStream;
 
@@ -98,91 +100,138 @@ public class ExportServiceImpl implements ExportService {
 
 final ZipOutputStream zipOutputStream = new ZipOutputStream(stream);
 
-//final AtomicInteger count = new AtomicInteger();
 final ApplicationScope appScope = 
exportRequestBuilder.getApplicationScope().get();
 final Observable applicationScopes = 
Observable.just(appScope);
 
-
-//final String jobId = StringUtils.sanitizeUUID( 
UUIDGenerator.newTimeUUID() );
 final EntityCollectionManager ecm = 
entityCollectionManagerFactory.createCollectionManager(appScope);
 
-final String rootPath = appScope.getApplication().getUuid().toString();
-
 GraphManager gm = managerCache.getGraphManager( appScope );
 
+final AtomicInteger entityFileCount = new AtomicInteger();
+final AtomicInteger connectionFileCount = new AtomicInteger();
+
 allEntityIdsObservable.getEdgesToEntities( applicationScopes, 
Optional.absent(), Optional.absent() )
-.doOnNext( edgeScope -> {
+.buffer( 1000 )
+.doOnNext( edgeScopes -> {
+
 
 try {
 
-// load the entity and convert to a normal map
-Entity entity = 
ecm.load(edgeScope.getEdge().getTargetNode()).toBlocking().lastOrDefault(null);
-Map entityMap = CpEntityMapUtils.toMap(entity);
+final String filenameWithPath = "entities/" +
+"entities."+ entityFileCount.get() + ".json";
+
+logger.debug("adding zip entry: {}", filenameWithPath);
+zipOutputStream.putNextEntry(new 
ZipEntry(filenameWithPath));
+
+edgeScopes.forEach( edgeScope -> {
+
+
+try {
+// load the entity and convert to a normal map
+Entity entity = 
ecm.load(edgeScope.getEdge().getTargetNode()).toBlocking().lastOrDefault(null);
+Map entityMap = CpEntityMapUtils.toMap(entity);
+
+if (entity != null) {
+
 
-if (entity != null) {
-final String filenameWithPath = rootPath + "/" +
-
edgeScope.getEdge().getSourceNode().getUuid().toString() + "_" +
-edgeScope.getEdge().getType() + "_" +
-
edgeScope.getEdge().getTargetNode().getUuid().toString() + ".json";
 
-logger.debug("adding zip entry: {}", filenameWithPath);
-zipOutputStream.putNextEntry(new 
ZipEntry(filenameWithPath));
+logger.debug("writing and flushing entity to 
zip stream: {}", jsonSerializer.toString(entityMap));
+
zipOutputStream.write(jsonSerializer.toString(entityMap).getBytes());
+zipOutputStream.write("\n".getBytes());
+
 
+} else {
+logger.warn("{}  did not have corresponding 
entity, not writing", edgeScope.toString());
+}
+
+} ca

[3/4] usergrid git commit: Update new ExportService to better stream out the results directly from the database. Added tests to validate

[mrusso]

Update new ExportService to better stream out the results directly from the 
database.  Added tests to validate


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/b37fc81e
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/b37fc81e
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/b37fc81e

Branch: refs/heads/master
Commit: b37fc81e984c43310bb8590a606ce639312e24b2
Parents: 2d56d81
Author: Michael Russo 
Authored: Thu Mar 30 19:53:12 2017 -0700
Committer: Michael Russo 
Committed: Thu Mar 30 19:53:12 2017 -0700

--
 .../corepersistence/export/ExportService.java   |   6 +-
 .../export/ExportServiceImpl.java   | 245 +++
 .../corepersistence/export/ExportServiceIT.java | 122 +++
 .../MvccEntitySerializationStrategyV3Impl.java  |   8 +-
 .../exceptions/AbstractExceptionMapper.java |   3 +-
 .../organizations/OrganizationResource.java | 190 +--
 .../applications/ApplicationResource.java   | 314 +--
 .../notifications/NotificationsService.java |  13 +-
 .../apns/NotificationsServiceIT.java|  49 ++-
 9 files changed, 558 insertions(+), 392 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/usergrid/blob/b37fc81e/stack/core/src/main/java/org/apache/usergrid/corepersistence/export/ExportService.java
--
diff --git 
a/stack/core/src/main/java/org/apache/usergrid/corepersistence/export/ExportService.java
 
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/export/ExportService.java
index 7615448..0c7789b 100644
--- 
a/stack/core/src/main/java/org/apache/usergrid/corepersistence/export/ExportService.java
+++ 
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/export/ExportService.java
@@ -30,15 +30,15 @@ public interface ExportService {
 
 
 /**
- * Perform an application export
+ * Perform an application export into the provided OutputStream
  *
  * @param exportRequestBuilder The builder to build the request
  */
-void export(final ExportRequestBuilder exportRequestBuilder, OutputStream 
stream) throws IOException;
+void export(final ExportRequestBuilder exportRequestBuilder, OutputStream 
stream) throws RuntimeException;
 
 
 /**
- * Generate a build for the index
+ * Generate a builder for the export request
  */
 ExportRequestBuilder getBuilder();
 

http://git-wip-us.apache.org/repos/asf/usergrid/blob/b37fc81e/stack/core/src/main/java/org/apache/usergrid/corepersistence/export/ExportServiceImpl.java
--
diff --git 
a/stack/core/src/main/java/org/apache/usergrid/corepersistence/export/ExportServiceImpl.java
 
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/export/ExportServiceImpl.java
index 47d6982..e561ac6 100644
--- 
a/stack/core/src/main/java/org/apache/usergrid/corepersistence/export/ExportServiceImpl.java
+++ 
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/export/ExportServiceImpl.java
@@ -31,6 +31,8 @@ import org.apache.usergrid.corepersistence.util.CpNamingUtils;
 import org.apache.usergrid.corepersistence.util.ObjectJsonSerializer;
 import org.apache.usergrid.persistence.collection.EntityCollectionManager;
 import 
org.apache.usergrid.persistence.collection.EntityCollectionManagerFactory;
+import org.apache.usergrid.persistence.collection.EntitySet;
+import 
org.apache.usergrid.persistence.collection.serialization.impl.EntitySetImpl;
 import org.apache.usergrid.persistence.core.scope.ApplicationScope;
 import org.apache.usergrid.persistence.graph.GraphManager;
 import org.apache.usergrid.persistence.graph.SearchByEdgeType;
@@ -41,6 +43,8 @@ import org.apache.usergrid.persistence.map.MapScope;
 import org.apache.usergrid.persistence.map.impl.MapScopeImpl;
 import org.apache.usergrid.persistence.model.entity.Entity;
 
+import org.apache.usergrid.persistence.model.entity.Id;
+import org.apache.usergrid.utils.InflectionUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import rx.Observable;
@@ -48,7 +52,9 @@ import rx.schedulers.Schedulers;
 
 
 import java.io.*;
+import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.zip.ZipEntry;
@@ -58,47 +64,32 @@ import java.util.zip.ZipOutputStream;
 @Singleton
 public class ExportServiceImpl implements ExportService {
 
-private static final Logger logger = LoggerFactory.getLogger( 
ReIndexServiceImpl.class );
-
-private static final MapScope RESUME_MAP_SCOPE =
-new MapScopeImpl( CpNamingUtils.getManagementApplicationId(), 
"export-status" );
-
-
-private static final String

[4/4] usergrid git commit: Fix issue where duplicate receipts were created when a notification failed.

[mrusso]

Fix issue where duplicate receipts were created when a notification failed.


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/8b63aae7
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/8b63aae7
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/8b63aae7

Branch: refs/heads/master
Commit: 8b63aae7def01cbf4f83e13bc5ef1b237344174b
Parents: b37fc81
Author: Michael Russo 
Authored: Fri Mar 31 00:40:39 2017 -0700
Committer: Michael Russo 
Committed: Fri Mar 31 00:40:39 2017 -0700

--
 .../org/apache/usergrid/services/notifications/TaskManager.java | 1 -
 1 file changed, 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/usergrid/blob/8b63aae7/stack/services/src/main/java/org/apache/usergrid/services/notifications/TaskManager.java
--
diff --git 
a/stack/services/src/main/java/org/apache/usergrid/services/notifications/TaskManager.java
 
b/stack/services/src/main/java/org/apache/usergrid/services/notifications/TaskManager.java
index 870cae9..678b88a 100644
--- 
a/stack/services/src/main/java/org/apache/usergrid/services/notifications/TaskManager.java
+++ 
b/stack/services/src/main/java/org/apache/usergrid/services/notifications/TaskManager.java
@@ -112,7 +112,6 @@ public class TaskManager {
 this.saveReceipt( notification, new SimpleEntityRef( 
Device.ENTITY_TYPE, deviceUUID ), receipt, true );
 }
 
-completed(notifier, deviceUUID);
 finishedBatch();
 
 } catch (Exception e){

usergrid git commit: Default disable the query breaker enforcement.

[mrusso]

Repository: usergrid
Updated Branches:
  refs/heads/master 89178326d -> ea1ba360d


Default disable the query breaker enforcement.


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/ea1ba360
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/ea1ba360
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/ea1ba360

Branch: refs/heads/master
Commit: ea1ba360da0a9986ef18ad2e8d559ca0c7df1be8
Parents: 8917832
Author: Michael Russo 
Authored: Sun Mar 26 11:39:08 2017 -0700
Committer: Michael Russo 
Committed: Sun Mar 26 11:39:08 2017 -0700

--
 .../main/java/org/apache/usergrid/persistence/index/IndexFig.java  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/usergrid/blob/ea1ba360/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
--
diff --git 
a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
 
b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
index 4091e5e..9b42da3 100644
--- 
a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
+++ 
b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
@@ -235,7 +235,7 @@ public interface IndexFig extends GuicyFig {
 long getQueryBreakerErrorIndexSizeBytes();
 
 
-@Default("true")
+@Default("false")
 @Key( USERGRID_QUERYANALYZER_ENFORCE )
 boolean enforceQueryBreaker();
 }

usergrid git commit: Fixes a handful of issues around single quotes and plus symbols in queries and email addresses. See: USERGRID-29 USERGRID-1041 USERGRID-1117 USERGRID-1338

[mrusso]

Repository: usergrid
Updated Branches:
  refs/heads/master d92110853 -> 89178326d


Fixes a handful of issues around single quotes and plus symbols in queries and 
email addresses. See: USERGRID-29 USERGRID-1041 USERGRID-1117 USERGRID-1338


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/89178326
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/89178326
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/89178326

Branch: refs/heads/master
Commit: 89178326debbd82660f9f2691b3b8cf06d847d4a
Parents: d921108
Author: Michael Russo 
Authored: Thu Mar 23 23:10:34 2017 -0700
Committer: Michael Russo 
Committed: Thu Mar 23 23:10:34 2017 -0700

--
 .../org/apache/usergrid/persistence/Query.java  |  1 +
 .../apache/usergrid/persistence/QueryTest.java  | 56 
 .../index/impl/EsEntityIndexImpl.java   |  2 +-
 .../persistence/index/impl/EsQueryVistor.java   |  7 ++-
 .../persistence/index/query/Identifier.java |  6 +--
 .../java/org/apache/usergrid/rest/BasicIT.java  | 33 
 .../collection/users/UserResourceIT.java| 48 +
 .../queries/SelectMappingsQueryTest.java| 49 +
 .../organizations/AdminEmailEncodingIT.java |  2 -
 9 files changed, 197 insertions(+), 7 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/usergrid/blob/89178326/stack/core/src/main/java/org/apache/usergrid/persistence/Query.java
--
diff --git 
a/stack/core/src/main/java/org/apache/usergrid/persistence/Query.java 
b/stack/core/src/main/java/org/apache/usergrid/persistence/Query.java
index 1015f76..900bda5 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/Query.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/Query.java
@@ -237,6 +237,7 @@ public class Query {
 }
 
 if ( ql != null ) {
+ql = ql.replace("+", "%2b"); // ql string supports literal + 
symbol, encode so it will decode correctly later
 q = Query.fromQL( decode( ql ) );
 }
 

http://git-wip-us.apache.org/repos/asf/usergrid/blob/89178326/stack/core/src/test/java/org/apache/usergrid/persistence/QueryTest.java
--
diff --git 
a/stack/core/src/test/java/org/apache/usergrid/persistence/QueryTest.java 
b/stack/core/src/test/java/org/apache/usergrid/persistence/QueryTest.java
new file mode 100644
index 000..38584dd
--- /dev/null
+++ b/stack/core/src/test/java/org/apache/usergrid/persistence/QueryTest.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.usergrid.persistence;
+
+
+import org.junit.Test;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.junit.Assert.assertEquals;
+
+public class QueryTest {
+
+@Test
+public void testQueryParamsWithPlus(){
+
+String qlString = "select * where email='test+va...@usergrid.com'";
+
+Map> queryParams = new HashMap<>();
+queryParams.put("ql", Collections.singletonList(qlString) );
+Query query = Query.fromQueryParams(queryParams);
+
+assertEquals(qlString, query.getQl().get());
+
+}
+
+@Test
+public void testQueryParamsWithUrlEncodedPlus(){
+
+String qlString = "select * where email='test+va...@usergrid.com'";
+Map> queryParams = new HashMap<>();
+queryParams.put("ql", Collections.singletonList(qlString.replace("+", 
"%2b")));
+Query query = Query.fromQueryParams(queryParams);
+
+assertEquals(qlString, query.getQl().get());
+
+}
+
+}

http://git-wip-us.apache.org/repos/asf/usergrid/blob/89178326/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
--
diff --git 
a/stack/corepersistence/queryindex/src/main/java/org/apac

[1/4] usergrid git commit: [USERGRID-1336] Initial commit of a simple query analyzer feature.

[mrusso]

Repository: usergrid
Updated Branches:
  refs/heads/master 3b1b0ca1c -> d92110853


http://git-wip-us.apache.org/repos/asf/usergrid/blob/a4941462/stack/rest/src/main/java/org/apache/usergrid/rest/exceptions/QueryAnalyzerEnforcementExceptionMapper.java
--
diff --git 
a/stack/rest/src/main/java/org/apache/usergrid/rest/exceptions/QueryAnalyzerEnforcementExceptionMapper.java
 
b/stack/rest/src/main/java/org/apache/usergrid/rest/exceptions/QueryAnalyzerEnforcementExceptionMapper.java
new file mode 100644
index 000..fed97d1
--- /dev/null
+++ 
b/stack/rest/src/main/java/org/apache/usergrid/rest/exceptions/QueryAnalyzerEnforcementExceptionMapper.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.usergrid.rest.exceptions;
+
+
+import 
org.apache.usergrid.persistence.index.exceptions.QueryAnalyzerEnforcementException;
+import org.apache.usergrid.rest.ApiResponse;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.ws.rs.core.Response;
+import javax.ws.rs.ext.Provider;
+
+import static javax.ws.rs.core.Response.Status.SERVICE_UNAVAILABLE;
+import static org.apache.usergrid.utils.JsonUtils.mapToJsonString;
+
+
+@Provider
+public class QueryAnalyzerEnforcementExceptionMapper extends 
AbstractExceptionMapper {
+
+private static final Logger logger = LoggerFactory.getLogger( 
QueryAnalyzerEnforcementExceptionMapper.class );
+
+@Override
+public Response toResponse( QueryAnalyzerEnforcementException e ) {
+
+// build a proper ApiResponse object
+ApiResponse apiResponse = new ApiResponse();
+apiResponse.setError("query_analyzer_violations_enforced");
+apiResponse.setErrorDescription(e.getErrorMessage());
+
+logger.warn(e.getErrorMessage());
+
+// give toResponse() the json string value of the ApiResponse
+// skip logging because we use a 5XX but it's not technically an error 
for the logs
+return toResponse( SERVICE_UNAVAILABLE, mapToJsonString(apiResponse), 
true );
+}
+}

http://git-wip-us.apache.org/repos/asf/usergrid/blob/a4941462/stack/rest/src/main/java/org/apache/usergrid/rest/exceptions/QueryAnalyzerExceptionMapper.java
--
diff --git 
a/stack/rest/src/main/java/org/apache/usergrid/rest/exceptions/QueryAnalyzerExceptionMapper.java
 
b/stack/rest/src/main/java/org/apache/usergrid/rest/exceptions/QueryAnalyzerExceptionMapper.java
new file mode 100644
index 000..bd5db14
--- /dev/null
+++ 
b/stack/rest/src/main/java/org/apache/usergrid/rest/exceptions/QueryAnalyzerExceptionMapper.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.usergrid.rest.exceptions;
+
+
+import org.apache.usergrid.persistence.index.QueryAnalyzer;
+import org.apache.usergrid.persistence.index.exceptions.QueryAnalyzerException;
+import org.apache.usergrid.rest.ApiResponse;
+
+import javax.ws.rs.core.MultivaluedHashMap;
+import javax.ws.rs.core.MultivaluedMap;
+import javax.ws.rs.core.Response;
+import javax.ws.rs.ext.Provider;
+
+import java.util.Collections;
+import java.util.HashMap;
+
+import static javax.ws.rs.core.Response.Status.OK;
+import static org.apache.usergrid.utils.JsonUtils.mapToFormattedJsonString;
+
+
+@Provider
+public class QueryAnalyzerExceptionMapper extends 
AbstractExceptionMapper {
+
+// Th

[3/4] usergrid git commit: Add query analyzer condition for full collection sorts when the collection size exceeds the limit. Add QueryAnalyzer tests.

[mrusso]

Add query analyzer condition for full collection sorts when the collection size 
exceeds the limit.  Add QueryAnalyzer tests.


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/f556e42b
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/f556e42b
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/f556e42b

Branch: refs/heads/master
Commit: f556e42b0fafb41a5674812a1f83235bc1008bde
Parents: a494146
Author: Michael Russo 
Authored: Sat Feb 25 15:43:10 2017 -0800
Committer: Michael Russo 
Committed: Sat Feb 25 15:43:10 2017 -0800

--
 .../usergrid/persistence/index/IndexFig.java|   2 +-
 .../persistence/index/QueryAnalyzer.java|  14 ++
 .../index/impl/QueryAnalyzerTest.java   | 169 +++
 3 files changed, 184 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/usergrid/blob/f556e42b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
--
diff --git 
a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
 
b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
index e23961c..4091e5e 100644
--- 
a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
+++ 
b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
@@ -225,7 +225,7 @@ public interface IndexFig extends GuicyFig {
 int getQueryBreakerErrorSortPredicateCount();
 
 
-@Default("5L") // 500 MB
+@Default("5") // 500 MB
 @Key(USERGRID_QUERYANALYZER_COLLECTIONSIZE)
 long getQueryBreakerErrorCollectionSizeBytes();
 

http://git-wip-us.apache.org/repos/asf/usergrid/blob/f556e42b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/QueryAnalyzer.java
--
diff --git 
a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/QueryAnalyzer.java
 
b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/QueryAnalyzer.java
index 86f1e5d..6d3cd92 100644
--- 
a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/QueryAnalyzer.java
+++ 
b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/QueryAnalyzer.java
@@ -31,6 +31,7 @@ public class QueryAnalyzer {
 public static final String v_operand_count = "operand_count_exceeded";
 public static final String v_large_collection = 
"large_collection_size_bytes";
 public static final String v_large_index = "large_index_size_bytes";
+public static final String v_full_collection_sort = "full_collection_sort";
 
 public static final String k_violation = "violation";
 public static final String k_limit = "limit";
@@ -69,6 +70,19 @@ public class QueryAnalyzer {
 put(k_limit, errorCollectionSizeBytes);
 put(k_actual, collectionSizeInBytes);
 }});
+
+// query like "select * order by created asc"
+if(parsedQuery.getSelectFieldMappings().size() < 1 &&
+
!parsedQuery.getOriginalQuery().toLowerCase().contains("where") &&
+parsedQuery.getSortPredicates().size() > 0 ){
+
+violations.add(new HashMap(3){{
+put(k_violation, v_full_collection_sort);
+put(k_limit, null);
+put(k_actual, null);
+}});
+}
+
 }
 
 // complex queries can be determined from the # of operands and sort 
predicates

http://git-wip-us.apache.org/repos/asf/usergrid/blob/f556e42b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/QueryAnalyzerTest.java
--
diff --git 
a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/QueryAnalyzerTest.java
 
b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/QueryAnalyzerTest.java
new file mode 100644
index 000..8c319c0
--- /dev/null
+++ 
b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/QueryAnalyzerTest.java
@@ -0,0 +1,169 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ *  contributor license agreements.  The ASF licenses this file to You
+ * under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   

[4/4] usergrid git commit: Merge commit 'refs/pull/567/head' of github.com:apache/usergrid

[mrusso]

Merge commit 'refs/pull/567/head' of github.com:apache/usergrid


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/d9211085
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/d9211085
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/d9211085

Branch: refs/heads/master
Commit: d92110853e17ab4a67b19ab8b0c28a48584883ce
Parents: 3b1b0ca f556e42
Author: Michael Russo 
Authored: Thu Mar 23 13:27:42 2017 -0700
Committer: Michael Russo 
Committed: Thu Mar 23 13:27:42 2017 -0700

--
 .../corepersistence/CpRelationManager.java  |   9 +
 .../pipeline/builder/IdBuilder.java |  10 +-
 .../pipeline/read/FilterFactory.java|   7 +-
 .../search/AbstractElasticSearchFilter.java |  21 ++-
 .../read/search/SearchCollectionFilter.java |   5 +-
 .../read/search/SearchConnectionFilter.java |   5 +-
 .../service/AggregationServiceImpl.java |   6 +-
 .../service/CollectionSearch.java   |  10 ++
 .../service/CollectionServiceImpl.java  |   2 +-
 .../service/ConnectionSearch.java   |  12 ++
 .../service/ConnectionServiceImpl.java  |   4 +-
 .../org/apache/usergrid/persistence/Query.java  |   9 +
 .../org/apache/usergrid/utils/JsonUtils.java|   2 +-
 .../corepersistence/StaleIndexCleanupTest.java  |   2 +-
 .../index/AsyncIndexServiceTest.java|   2 +-
 .../corepersistence/index/IndexServiceTest.java |   8 +-
 .../usergrid/persistence/index/EntityIndex.java |  11 +-
 .../usergrid/persistence/index/IndexFig.java|  39 +
 .../persistence/index/QueryAnalyzer.java| 143 
 .../QueryAnalyzerEnforcementException.java  |  45 +
 .../exceptions/QueryAnalyzerException.java  |  57 +++
 .../index/impl/EsEntityIndexImpl.java   |  61 ++-
 .../persistence/index/impl/EntityIndexTest.java |  86 +-
 .../persistence/index/impl/GeoPagingTest.java   |   2 +-
 .../index/impl/IndexLoadTestsIT.java|   2 +-
 .../index/impl/QueryAnalyzerTest.java   | 169 +++
 .../rest/applications/CollectionResource.java   |  17 --
 .../rest/applications/ServiceResource.java  |   9 +-
 .../exceptions/AbstractExceptionMapper.java |  17 +-
 ...QueryAnalyzerEnforcementExceptionMapper.java |  51 ++
 .../QueryAnalyzerExceptionMapper.java   |  69 
 .../usergrid/services/AbstractService.java  |   2 +
 .../usergrid/services/ServiceManager.java   |  15 +-
 .../usergrid/services/ServiceRequest.java   |  26 ++-
 34 files changed, 809 insertions(+), 126 deletions(-)
--

[2/4] usergrid git commit: [USERGRID-1336] Initial commit of a simple query analyzer feature.

[mrusso]

[USERGRID-1336] Initial commit of a simple query analyzer feature.


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/a4941462
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/a4941462
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/a4941462

Branch: refs/heads/master
Commit: a494146250ea92c6d0ebbb9c579ff194525f5eed
Parents: f8e644d
Author: Michael Russo 
Authored: Fri Feb 24 14:49:09 2017 -0800
Committer: Michael Russo 
Committed: Fri Feb 24 14:49:09 2017 -0800

--
 .../corepersistence/CpRelationManager.java  |   9 ++
 .../pipeline/builder/IdBuilder.java |  10 +-
 .../pipeline/read/FilterFactory.java|   7 +-
 .../search/AbstractElasticSearchFilter.java |  21 ++-
 .../read/search/SearchCollectionFilter.java |   5 +-
 .../read/search/SearchConnectionFilter.java |   5 +-
 .../service/AggregationServiceImpl.java |   6 +-
 .../service/CollectionSearch.java   |  10 ++
 .../service/CollectionServiceImpl.java  |   2 +-
 .../service/ConnectionSearch.java   |  12 ++
 .../service/ConnectionServiceImpl.java  |   4 +-
 .../org/apache/usergrid/persistence/Query.java  |   9 ++
 .../org/apache/usergrid/utils/JsonUtils.java|   2 +-
 .../corepersistence/StaleIndexCleanupTest.java  |   2 +-
 .../index/AsyncIndexServiceTest.java|   2 +-
 .../corepersistence/index/IndexServiceTest.java |   8 +-
 .../usergrid/persistence/index/EntityIndex.java |  11 +-
 .../usergrid/persistence/index/IndexFig.java|  39 ++
 .../persistence/index/QueryAnalyzer.java| 129 +++
 .../QueryAnalyzerEnforcementException.java  |  45 +++
 .../exceptions/QueryAnalyzerException.java  |  57 
 .../index/impl/EsEntityIndexImpl.java   |  61 -
 .../persistence/index/impl/EntityIndexTest.java |  86 ++---
 .../persistence/index/impl/GeoPagingTest.java   |   2 +-
 .../index/impl/IndexLoadTestsIT.java|   2 +-
 .../rest/applications/CollectionResource.java   |  17 ---
 .../rest/applications/ServiceResource.java  |   9 +-
 .../exceptions/AbstractExceptionMapper.java |  17 ++-
 ...QueryAnalyzerEnforcementExceptionMapper.java |  51 
 .../QueryAnalyzerExceptionMapper.java   |  69 ++
 .../usergrid/services/AbstractService.java  |   2 +
 .../usergrid/services/ServiceManager.java   |  15 +--
 .../usergrid/services/ServiceRequest.java   |  26 ++--
 33 files changed, 626 insertions(+), 126 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/usergrid/blob/a4941462/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
--
diff --git 
a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
 
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
index 4a759ac..c02ca7d 100644
--- 
a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
+++ 
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
@@ -629,6 +629,7 @@ public class CpRelationManager implements RelationManager {
 final Query toExecute = adjustQuery( query );
 final Optional queryString = query.isGraphSearch()? 
Optional.absent(): query.getQl();
 final Id ownerId = headEntity.asId();
+final boolean analyzeOnly = query.getAnalyzeOnly();
 
 
 if(query.getLevel() == Level.IDS ){
@@ -642,6 +643,8 @@ public class CpRelationManager implements RelationManager {
 new CollectionSearch( applicationScope, ownerId, 
collectionName, collection.getType(), toExecute.getLimit(),
 queryString, cursor );
 
+search.setAnalyzeOnly(analyzeOnly);
+
 return collectionService.searchCollectionIds( search );
 }
 }.next();
@@ -658,6 +661,8 @@ public class CpRelationManager implements RelationManager {
 new CollectionSearch( applicationScope, ownerId, 
collectionName, collection.getType(), toExecute.getLimit(),
 queryString, cursor );
 
+search.setAnalyzeOnly(analyzeOnly);
+
 return collectionService.searchCollection( search );
 }
 }.next();
@@ -919,6 +924,8 @@ public class CpRelationManager implements RelationManager {
 
 headEntity = em.validate( headEntity );
 
+final boolean analyzeOnly = query.getAnalyzeOnly();
+
 
 final Query toExecute = adjustQuery( query );
 
@@ -951,6 +958,7 @@ public class CpRelationManager implements RelationManager {
 final ConnectionSearch sea

usergrid git commit: Fix issue where default passwords are not being created when not provided ( you are allowed to create new admin users with just an email and it will invoke a reset password flow).

[mrusso]

Repository: usergrid
Updated Branches:
  refs/heads/master 9141f8dc2 -> 3b1b0ca1c


Fix issue where default passwords are not being created when not provided ( you 
are allowed to create new admin users with just an email and it will invoke a 
reset password flow).


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/3b1b0ca1
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/3b1b0ca1
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/3b1b0ca1

Branch: refs/heads/master
Commit: 3b1b0ca1c093d03074c8dff86cf2d14deea96f79
Parents: 9141f8d
Author: Michael Russo 
Authored: Thu Mar 23 13:18:19 2017 -0700
Committer: Michael Russo 
Committed: Thu Mar 23 13:18:19 2017 -0700

--
 .../rest/management/OrganizationsIT.java| 32 
 .../cassandra/ManagementServiceImpl.java|  9 +++---
 2 files changed, 36 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/usergrid/blob/3b1b0ca1/stack/rest/src/test/java/org/apache/usergrid/rest/management/OrganizationsIT.java
--
diff --git 
a/stack/rest/src/test/java/org/apache/usergrid/rest/management/OrganizationsIT.java
 
b/stack/rest/src/test/java/org/apache/usergrid/rest/management/OrganizationsIT.java
index 29841a0..2bbdaaf 100644
--- 
a/stack/rest/src/test/java/org/apache/usergrid/rest/management/OrganizationsIT.java
+++ 
b/stack/rest/src/test/java/org/apache/usergrid/rest/management/OrganizationsIT.java
@@ -92,6 +92,38 @@ public class OrganizationsIT extends AbstractRestIT {
 assertEquals( "Apigee", returnedUser.getProperties().get( "company" ) 
);
 }
 
+// Admin users are allowed to be created with an email address only, in 
which a password reset flow is invoked
+@Test
+public void createNewOrgUserNoPassword() throws Exception {
+
+//User property to see if owner properties exist when created.
+Map userProperties = new HashMap();
+userProperties.put( "company", "Usergrid" );
+
+//Create organization
+Organization organization = createOrgPayload( 
"createNewOrgUserNoPassword", userProperties );
+
+//Get back organization response
+Organization organizationResponse = 
clientSetup.getRestClient().management().orgs().post( organization );
+
+assertNotNull( organizationResponse );
+
+
+//Creates token
+Token token =
+
clientSetup.getRestClient().management().token().post(false,Token.class, new 
Token( "password",
+organization.getUsername(), organization.getPassword() ) 
,null);
+this.management().token().setToken(token);
+
+assertNotNull( token );
+
+
+Map newUser = new HashMap(){{put("email", "e...@usergrid.com");}};
+clientSetup.getRestClient().management().orgs().org( 
organization.getOrganization() ).users().post(newUser);
+
+// the test will fail if any exception is thrown
+}
+
 
 /**
  * Creates a organization with an owner, then attempts to create an 
organization with the same name ( making sure it

http://git-wip-us.apache.org/repos/asf/usergrid/blob/3b1b0ca1/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java
--
diff --git 
a/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java
 
b/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java
index c0915e4..2ba9bde 100644
--- 
a/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java
+++ 
b/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java
@@ -949,7 +949,7 @@ public class ManagementServiceImpl implements 
ManagementService {
 @Override
 public UserInfo createAdminFrom( UUID organizationId, User user, String 
password ) throws Exception {
 
-Collection policyVioliations = passwordPolicy.policyCheck( 
password, false );
+Collection policyVioliations = passwordPolicy.policyCheck( 
password, true );
 if ( !policyVioliations.isEmpty() ) {
 throw new PasswordPolicyViolationException( 
passwordPolicy.getDescription( true ), policyVioliations );
 }
@@ -1011,11 +1011,10 @@ public class ManagementServiceImpl implements 
ManagementService {
 throws Exception {
 
 
-logger.info( "createAdminUserInternal: {}", username );
+logger.debug( "createAdminUserInternal - username: {}, email: {}, 
name: {}", username, email, name );
 
-Collection policyVioliations = passwordPolicy.policyCheck( 
password, true );
-if ( !policyVioliations.isEmpty() ) {
- 

[2/2] usergrid git commit: Merge branch 'master' of https://git-wip-us.apache.org/repos/asf/usergrid

[mrusso]

Merge branch 'master' of https://git-wip-us.apache.org/repos/asf/usergrid


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/9141f8dc
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/9141f8dc
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/9141f8dc

Branch: refs/heads/master
Commit: 9141f8dc2d8ab44cb739b98836e41e2632caa961
Parents: f6aed13 104f64e
Author: Michael Russo 
Authored: Tue Mar 21 08:47:06 2017 -0700
Committer: Michael Russo 
Committed: Tue Mar 21 08:47:06 2017 -0700

--
 stack/corepersistence/pom.xml | 1 +
 1 file changed, 1 insertion(+)
--