Repository: usergrid Updated Branches: refs/heads/master d92110853 -> 89178326d
Fixes a handful of issues around single quotes and plus symbols in queries and email addresses. See: USERGRID-29 USERGRID-1041 USERGRID-1117 USERGRID-1338 Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/89178326 Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/89178326 Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/89178326 Branch: refs/heads/master Commit: 89178326debbd82660f9f2691b3b8cf06d847d4a Parents: d921108 Author: Michael Russo <[email protected]> Authored: Thu Mar 23 23:10:34 2017 -0700 Committer: Michael Russo <[email protected]> Committed: Thu Mar 23 23:10:34 2017 -0700 ---------------------------------------------------------------------- .../org/apache/usergrid/persistence/Query.java | 1 + .../apache/usergrid/persistence/QueryTest.java | 56 ++++++++++++++++++++ .../index/impl/EsEntityIndexImpl.java | 2 +- .../persistence/index/impl/EsQueryVistor.java | 7 ++- .../persistence/index/query/Identifier.java | 6 +-- .../java/org/apache/usergrid/rest/BasicIT.java | 33 ++++++++++++ .../collection/users/UserResourceIT.java | 48 +++++++++++++++++ .../queries/SelectMappingsQueryTest.java | 49 +++++++++++++++++ .../organizations/AdminEmailEncodingIT.java | 2 - 9 files changed, 197 insertions(+), 7 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/usergrid/blob/89178326/stack/core/src/main/java/org/apache/usergrid/persistence/Query.java ---------------------------------------------------------------------- diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/Query.java b/stack/core/src/main/java/org/apache/usergrid/persistence/Query.java index 1015f76..900bda5 100644 --- a/stack/core/src/main/java/org/apache/usergrid/persistence/Query.java +++ b/stack/core/src/main/java/org/apache/usergrid/persistence/Query.java @@ -237,6 +237,7 @@ public class Query { } if ( ql != null ) { + ql = ql.replace("+", "%2b"); // ql string supports literal + symbol, encode so it will decode correctly later q = Query.fromQL( decode( ql ) ); } http://git-wip-us.apache.org/repos/asf/usergrid/blob/89178326/stack/core/src/test/java/org/apache/usergrid/persistence/QueryTest.java ---------------------------------------------------------------------- diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/QueryTest.java b/stack/core/src/test/java/org/apache/usergrid/persistence/QueryTest.java new file mode 100644 index 0000000..38584dd --- /dev/null +++ b/stack/core/src/test/java/org/apache/usergrid/persistence/QueryTest.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.usergrid.persistence; + + +import org.junit.Test; + +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.junit.Assert.assertEquals; + +public class QueryTest { + + @Test + public void testQueryParamsWithPlus(){ + + String qlString = "select * where email='[email protected]'"; + + Map<String,List<String>> queryParams = new HashMap<>(); + queryParams.put("ql", Collections.singletonList(qlString) ); + Query query = Query.fromQueryParams(queryParams); + + assertEquals(qlString, query.getQl().get()); + + } + + @Test + public void testQueryParamsWithUrlEncodedPlus(){ + + String qlString = "select * where email='[email protected]'"; + Map<String,List<String>> queryParams = new HashMap<>(); + queryParams.put("ql", Collections.singletonList(qlString.replace("+", "%2b"))); + Query query = Query.fromQueryParams(queryParams); + + assertEquals(qlString, query.getQl().get()); + + } + +} http://git-wip-us.apache.org/repos/asf/usergrid/blob/89178326/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java index 26e8f68..a35921c 100644 --- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java +++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java @@ -476,7 +476,7 @@ public class EsEntityIndexImpl implements EntityIndex,VersionedData { List<Map<String, Object>> violations = QueryAnalyzer.analyze(parsedQuery, totalEdgeSizeInBytes, totalIndexSizeInBytes, indexFig); if(indexFig.enforceQueryBreaker() && violations.size() > 0){ throw new QueryAnalyzerEnforcementException(violations, parsedQuery.getOriginalQuery()); - }else{ + }else if (violations.size() > 0){ logger.warn( QueryAnalyzer.violationsAsString(violations, parsedQuery.getOriginalQuery()) ); } http://git-wip-us.apache.org/repos/asf/usergrid/blob/89178326/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsQueryVistor.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsQueryVistor.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsQueryVistor.java index 7ee8961..4dd0d24 100644 --- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsQueryVistor.java +++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsQueryVistor.java @@ -363,7 +363,7 @@ public class EsQueryVistor implements QueryVisitor { //special case so we support our '*' char with wildcard, also should work for uuids if ( value instanceof String || value instanceof UUID ) { - final String stringValue = ((value instanceof String) ? (String)value : value.toString()).toLowerCase().trim(); + String stringValue = ((value instanceof String) ? (String)value : value.toString()).toLowerCase().trim(); // or field is just a string that does need a prefix us a query if ( stringValue.contains( "*" ) ) { @@ -377,6 +377,11 @@ public class EsQueryVistor implements QueryVisitor { return; } + // Usergrid query parser allows single quotes to be escaped in values + if ( stringValue.contains("\\'")) { + stringValue = stringValue.replace("\\'", "'"); + } + //it's an exact match, use a filter final TermFilterBuilder termFilter = FilterBuilders.termFilter( IndexingUtils.FIELD_STRING_NESTED_UNANALYZED, stringValue ); http://git-wip-us.apache.org/repos/asf/usergrid/blob/89178326/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/Identifier.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/Identifier.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/Identifier.java index f98ccc7..84a28f0 100644 --- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/Identifier.java +++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/Identifier.java @@ -31,9 +31,9 @@ import com.fasterxml.jackson.annotation.JsonIgnore; public class Identifier implements Serializable { - public static final String UUID_REX = + public static final String UUID_REX = "[A-Fa-f0-9]{8}-[A-Fa-f0-9]{4}-[A-Fa-f0-9]{4}-[A-Fa-f0-9]{4}-[A-Fa-f0-9]{12}"; - public static final String EMAIL_REX = "[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]{2,}"; + public static final String EMAIL_REX = "[a-zA-Z0-9._%'+\\-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]{2,}"; public enum Type { UUID, NAME, EMAIL @@ -46,7 +46,7 @@ public class Identifier implements Serializable { static Pattern emailRegEx = Pattern.compile( EMAIL_REX ); // "Pattern nameRegEx" below used to be [a-zA-Z0-9_\\-./], changed it to contain a 'space' to a // ddress https://issues.apache.org/jira/browse/USERGRID-94 - static Pattern nameRegEx = Pattern.compile( "[a-zA-Z0-9_\\-./ ]*" ); + static Pattern nameRegEx = Pattern.compile( "[a-zA-Z0-9_\\-./'+ ]*" ); private Identifier( Type type, Object value ) { http://git-wip-us.apache.org/repos/asf/usergrid/blob/89178326/stack/rest/src/test/java/org/apache/usergrid/rest/BasicIT.java ---------------------------------------------------------------------- diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/BasicIT.java b/stack/rest/src/test/java/org/apache/usergrid/rest/BasicIT.java index 905c522..997f56c 100644 --- a/stack/rest/src/test/java/org/apache/usergrid/rest/BasicIT.java +++ b/stack/rest/src/test/java/org/apache/usergrid/rest/BasicIT.java @@ -21,6 +21,7 @@ import org.apache.usergrid.persistence.index.utils.UUIDUtils; import org.apache.usergrid.rest.test.resource.AbstractRestIT; import org.apache.usergrid.rest.test.resource.model.ApiResponse; import org.apache.usergrid.rest.test.resource.model.Entity; +import org.apache.usergrid.rest.test.resource.model.QueryParameters; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -70,6 +71,38 @@ public class BasicIT extends AbstractRestIT { } @Test + public void testEntityWithSingleQuoteName() throws Exception { + + Entity payload = new Entity(); + payload.put( "name", "test'value" ); + + //Create a user with the payload + Entity returnedUser = this.app().collection( "suspects" ).post(payload); + assertNotNull( returnedUser ); + + + returnedUser = this.app().collection( "suspects" ).entity("test'value").get(); + + assertNotNull( returnedUser ); + } + + @Test + public void testEntityWithPlusName() throws Exception { + + Entity payload = new Entity(); + payload.put( "name", "test+value" ); + + //Create a user with the payload + Entity returnedUser = this.app().collection( "suspects" ).post(payload); + assertNotNull( returnedUser ); + + + returnedUser = this.app().collection( "suspects" ).entity("test+value").get(); + + assertNotNull( returnedUser ); + } + + @Test public void serviceResourceNotFoundReturns404() throws Exception { try { http://git-wip-us.apache.org/repos/asf/usergrid/blob/89178326/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/users/UserResourceIT.java ---------------------------------------------------------------------- diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/users/UserResourceIT.java b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/users/UserResourceIT.java index c9dc0d8..f8cb9d4e 100644 --- a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/users/UserResourceIT.java +++ b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/users/UserResourceIT.java @@ -303,6 +303,54 @@ public class UserResourceIT extends AbstractRestIT { } + @Test + public void getUserWithEmailContainingSingleQuote() throws IOException { + UUID id = UUIDUtils.newTimeUUID(); + + String username = "username-email" + "@usergrid.org"; + String name = "name" + id; + String email = "e'mail" + id + "@usergrid.org"; + + User map = new User(username, name, email, null); + map.put("email", email); + + usersResource.post(new Entity(map)); + refreshIndex(); + + // get the user with email property value + // get the user with username property that has an email value + Entity testUser = usersResource.entity(email).get(); + + assertEquals(username, testUser.get("username").toString()); + assertEquals(name, testUser.get("name").toString()); + assertEquals(email, testUser.get("email").toString()); + + } + + @Test + public void getUserWithEmailContainingPlus() throws IOException { + UUID id = UUIDUtils.newTimeUUID(); + + String username = "username-email" + "@usergrid.org"; + String name = "name" + id; + String email = "e+mail" + id + "@usergrid.org"; + + User map = new User(username, name, email, null); + map.put("email", email); + + usersResource.post(new Entity(map)); + refreshIndex(); + + // get the user with email property value + // get the user with username property that has an email value + Entity testUser = usersResource.entity(email).get(); + + assertEquals(username, testUser.get("username").toString()); + assertEquals(name, testUser.get("name").toString()); + assertEquals(email, testUser.get("email").toString()); + + } + /** * Tests that when querying all users, we get the same result size when using "order by" http://git-wip-us.apache.org/repos/asf/usergrid/blob/89178326/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/SelectMappingsQueryTest.java ---------------------------------------------------------------------- diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/SelectMappingsQueryTest.java b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/SelectMappingsQueryTest.java index fd33c15..286c984 100644 --- a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/SelectMappingsQueryTest.java +++ b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/SelectMappingsQueryTest.java @@ -98,6 +98,55 @@ public class SelectMappingsQueryTest extends QueryTestBase { assertEquals( 1, things.getNumOfEntities() ); } + @Test + public void testStringWithSingleQuote() throws Exception { + + String collectionName = "things"; + + String value = "test'value"; + String escapedValue = "test\\'value"; + + // create entity with testProp=value + Entity entity = new Entity() + .withProp( "testprop", value ); + app().collection( collectionName ).post( entity ); + refreshIndex(); + + // testProp and TESTPROP should now have otherValue + + QueryParameters params = new QueryParameters() + .setQuery( "select * where testprop='" + escapedValue + "'" ); + Collection things = this.app().collection( "things" ).get( params ); + assertEquals( 1, things.getNumOfEntities() ); + } + + @Test + public void testStringWithPlus() throws Exception { + + String collectionName = "things"; + String value = "[email protected]"; + + // create entity with value containing a plus symbol + Entity entity = new Entity() + .withProp( "testprop", value ); + app().collection( collectionName ).post( entity ); + refreshIndex(); + + // now query this without encoding the plus symbol + QueryParameters params = new QueryParameters() + .setQuery( "select * where testprop='" + value + "'" ); + Collection things = this.app().collection( "things" ).get( params ); + assertEquals( 1, things.getNumOfEntities() ); + + // again query with the plus symbol url encoded + String escapedValue = "ed%[email protected]"; + params = new QueryParameters() + .setQuery( "select * where testprop='" + escapedValue + "'" ); + things = this.app().collection( "things" ).get( params ); + assertEquals( 1, things.getNumOfEntities() ); + + } + /** * Field named testProp can be over-written by field named TESTPROP. http://git-wip-us.apache.org/repos/asf/usergrid/blob/89178326/stack/rest/src/test/java/org/apache/usergrid/rest/management/organizations/AdminEmailEncodingIT.java ---------------------------------------------------------------------- diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/management/organizations/AdminEmailEncodingIT.java b/stack/rest/src/test/java/org/apache/usergrid/rest/management/organizations/AdminEmailEncodingIT.java index b188aba..0f1a794 100644 --- a/stack/rest/src/test/java/org/apache/usergrid/rest/management/organizations/AdminEmailEncodingIT.java +++ b/stack/rest/src/test/java/org/apache/usergrid/rest/management/organizations/AdminEmailEncodingIT.java @@ -77,8 +77,6 @@ public class AdminEmailEncodingIT extends AbstractRestIT { * @throws Exception */ @Test - @Ignore("Pending https://issues.apache.org/jira/browse/USERGRID-1117";) - // This fails. I'm not sure if it is by design, but a single quote is valid in an email address public void getTokenQuote() throws Exception { doTest("'"); }
