check trailing page
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/1f83f0fa Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/1f83f0fa Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/1f83f0fa Branch: refs/heads/USERGRID-593 Commit: 1f83f0fabcac3281024637300978ff8eb7ec366e Parents: 0db7f4c Author: Shawn Feldman <sfeld...@apache.org> Authored: Fri Apr 17 16:43:31 2015 -0600 Committer: Shawn Feldman <sfeld...@apache.org> Committed: Fri Apr 17 16:43:31 2015 -0600 ---------------------------------------------------------------------- .../query/OrderByDiscardPagesPagingIT.java | 164 ------------------- .../persistence/index/CandidateResults.java | 9 +- .../impl/EsApplicationEntityIndexImpl.java | 2 +- .../persistence/index/impl/EntityIndexTest.java | 17 +- 4 files changed, 20 insertions(+), 172 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/1f83f0fa/stack/core/src/test/java/org/apache/usergrid/persistence/query/OrderByDiscardPagesPagingIT.java ---------------------------------------------------------------------- diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/query/OrderByDiscardPagesPagingIT.java b/stack/core/src/test/java/org/apache/usergrid/persistence/query/OrderByDiscardPagesPagingIT.java deleted file mode 100644 index 2cee402..0000000 --- a/stack/core/src/test/java/org/apache/usergrid/persistence/query/OrderByDiscardPagesPagingIT.java +++ /dev/null @@ -1,164 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.usergrid.persistence.query; - - -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.UUID; - -import org.apache.usergrid.persistence.index.query.Query; -import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import org.apache.usergrid.persistence.Entity; -import org.apache.usergrid.persistence.Results; -import org.apache.usergrid.persistence.cassandra.QueryProcessor; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - - -/** - * - */ -public class OrderByDiscardPagesPagingIT extends IteratingQueryIT { - - private static final Logger LOG = LoggerFactory.getLogger( OrderByDiscardPagesPagingIT.class ); - - private static final String search = "select * where field1 = true AND field2 = true order by ordinal"; - - private static final int PAGE_SIZE = 300; - - - @Test - public void testUnionPagingCollection() throws Exception { - - - final CollectionIoHelper collectionIoHelper = new CollectionIoHelper( app ); - - Set<UUID> created = performSetup( collectionIoHelper ); - - - testUnionPaging( collectionIoHelper, search, created ); - } - - - @Test - public void testUnionPagingConnection() throws Exception { - - final ConnectionHelper connectionHelper = new ConnectionHelper( app ); - - Set<UUID> created = performSetup( connectionHelper ); - - - testUnionPaging( connectionHelper, search, created ); - } - - - private Set<UUID> performSetup( final IoHelper io ) throws Exception { - io.doSetup(); - - int size = ( int ) ( QueryProcessor.PAGE_SIZE * 2.5 ); - - long start = System.currentTimeMillis(); - - LOG.info( "Writing {} entities.", size ); - - Set<UUID> entites = new HashSet<UUID>( size ); - - for ( int i = 0; i < size; i++ ) { - - Map<String, Object> entity = new HashMap<String, Object>(); - - entity.put( "ordinal", i ); - - int segment = i / PAGE_SIZE; - - boolean shouldBeReturned = segment % 2 != 0; - - if ( shouldBeReturned ) { - - entity.put( "field1", true ); - entity.put( "field2", true ); - } - else { - entity.put( "field1", false ); - entity.put( "field2", false ); - } - - - Entity saved = io.writeEntity( entity ); - - LOG.info( "Writing entity with id '{}'", saved.getUuid() ); - - if ( shouldBeReturned ) { - entites.add( saved.getUuid() ); - } - } - - long stop = System.currentTimeMillis(); - - LOG.info( "Writes took {} ms", stop - start ); - - return entites ; - } - - - private void testUnionPaging( final IoHelper io, final String queryString, final Set<UUID> expectedResults ) - throws Exception { - - //our field1Or has a result size < our page size, so it shouldn't blow up when the cursor is getting created - //the leaf iterator should insert it's own "no value left" into the cursor - Query query = Query.fromQL( queryString ); - query.setLimit( PAGE_SIZE ); - - Results results; - - long start = System.currentTimeMillis(); - - do { - - // now do simple ordering, should be returned in order - results = io.getResults( query ); - - final List<Entity> entities = results.getEntities(); - - for ( int i = 0; i < entities.size(); i++ ) { - final UUID uuid = entities.get( i ).getUuid(); - - assertTrue( "Value should not be returned twice", expectedResults.contains( uuid ) ); - - expectedResults.remove( uuid ); - } - - query.setCursor( results.getCursor() ); - } - while ( results.getCursor() != null ); - - long stop = System.currentTimeMillis(); - - LOG.info( "Query took {} ms to return {} entities", stop - start, expectedResults.size() ); - - assertEquals( "All entities returned", 0, expectedResults.size() ); - } -} http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/1f83f0fa/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/CandidateResults.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/CandidateResults.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/CandidateResults.java index f0955ea..14130bc 100644 --- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/CandidateResults.java +++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/CandidateResults.java @@ -49,12 +49,16 @@ public class CandidateResults implements Iterable<CandidateResult> { private final List<CandidateResult> candidates; private final Collection<SelectFieldMapping> getFieldMappings; + private final String esCursor; public CandidateResults( List<CandidateResult> candidates, - final Collection<SelectFieldMapping> getFieldMappings ) { + final Collection<SelectFieldMapping> getFieldMappings, + final String esCursor + ) { this.candidates = candidates; this.getFieldMappings = getFieldMappings; + this.esCursor = esCursor; } @@ -109,4 +113,7 @@ public class CandidateResults implements Iterable<CandidateResult> { return candidates.iterator(); } + public String getEsCursor() { + return esCursor; + } } http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/1f83f0fa/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java index 6574985..5b05b27 100644 --- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java +++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java @@ -310,8 +310,8 @@ public class EsApplicationEntityIndexImpl implements ApplicationEntityIndex { candidates.add( candidateResult ); } - final CandidateResults candidateResults = new CandidateResults( candidates, query.getSelectFieldMappings()); final String esScrollCursor = searchResponse.getScrollId(); + final CandidateResults candidateResults = new CandidateResults( candidates, query.getSelectFieldMappings(),esScrollCursor); // >= seems odd. However if we get an overflow, we need to account for it. if ( esScrollCursor != null && length >= limit ) { http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/1f83f0fa/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java index e95c2bf..9b8a9f5 100644 --- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java +++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java @@ -641,6 +641,7 @@ public class EntityIndexTest extends BaseIT { @Test public void testCursorFormat() throws Exception { + String myType = UUID.randomUUID().toString(); Id appId = new SimpleId( "application" ); Id ownerId = new SimpleId( "owner" ); @@ -655,7 +656,7 @@ public class EntityIndexTest extends BaseIT { final EntityIndexBatch batch = entityIndex.createBatch(); - final int size = 10; + final int size = 100; final List<Id> entityIds = new ArrayList<>( size ); @@ -671,6 +672,7 @@ public class EntityIndexTest extends BaseIT { put( "email", "e...@anuff.com" ); put( "middlename", middleName ); put( "ordinal", ordinal ); + put( "mytype", myType); }}; final Id userId = new SimpleId( "user" ); @@ -691,17 +693,18 @@ public class EntityIndexTest extends BaseIT { ei.refreshAsync().toBlocking().last(); - final int limit = 1; + final int limit = 5; final int expectedPages = size / limit; String cursor = null; + UUID lastId = null; for ( int i = 0; i < expectedPages; i++ ) { //** - final String query = "select * order by ordinal asc"; + final String query = "select * where mytype='"+myType+"' order by ordinal asc"; final CandidateResults results = cursor == null ? entityIndex.search( indexEdge, SearchTypes.allTypes(), query, limit ) : @@ -713,10 +716,12 @@ public class EntityIndexTest extends BaseIT { assertEquals( "Should be 16 bytes as hex", 32, cursor.length() ); - assertEquals( 1, results.size() ); + assertEquals(limit, results.size()); - - assertEquals( results.get( 0 ).getId(), entityIds.get( i ) ); + int ordinal = 0;//i == 0 ? 0 : 1; + assertNotEquals("Scroll matches last item from previous page",lastId, results.get(ordinal).getId().getUuid()); + lastId = results.get((i+1)*limit-1).getId().getUuid(); + assertEquals("Failed on page "+i ,results.get( ordinal ).getId(), entityIds.get( i*limit ) ); } //get our next page, we shouldn't get a cursor
