METAMODEL-1128: Fixed Fixes #136
Project: http://git-wip-us.apache.org/repos/asf/metamodel/repo Commit: http://git-wip-us.apache.org/repos/asf/metamodel/commit/f18ae8b7 Tree: http://git-wip-us.apache.org/repos/asf/metamodel/tree/f18ae8b7 Diff: http://git-wip-us.apache.org/repos/asf/metamodel/diff/f18ae8b7 Branch: refs/heads/5.x Commit: f18ae8b75f5baf4832a3c1942a9fd82373fa7355 Parents: 11710af Author: kaspersorensen <i.am.kasper.soren...@gmail.com> Authored: Fri Nov 11 13:38:28 2016 -0800 Committer: kaspersorensen <i.am.kasper.soren...@gmail.com> Committed: Fri Nov 11 13:38:28 2016 -0800 ---------------------------------------------------------------------- .../rest/ElasticSearchRestDataContext.java | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/metamodel/blob/f18ae8b7/elasticsearch/rest/src/main/java/org/apache/metamodel/elasticsearch/rest/ElasticSearchRestDataContext.java ---------------------------------------------------------------------- diff --git a/elasticsearch/rest/src/main/java/org/apache/metamodel/elasticsearch/rest/ElasticSearchRestDataContext.java b/elasticsearch/rest/src/main/java/org/apache/metamodel/elasticsearch/rest/ElasticSearchRestDataContext.java index b55db13..920217f 100644 --- a/elasticsearch/rest/src/main/java/org/apache/metamodel/elasticsearch/rest/ElasticSearchRestDataContext.java +++ b/elasticsearch/rest/src/main/java/org/apache/metamodel/elasticsearch/rest/ElasticSearchRestDataContext.java @@ -95,6 +95,9 @@ public class ElasticSearchRestDataContext extends QueryPostprocessDataContext im // 1 minute timeout public static final String TIMEOUT_SCROLL = "1m"; + // we scroll when more than 400 rows are expected + private static final int SCROLL_THRESHOLD = 400; + private final JestClient elasticSearchClient; private final String indexName; @@ -265,13 +268,18 @@ public class ElasticSearchRestDataContext extends QueryPostprocessDataContext im if (queryBuilder != null) { // where clause can be pushed down to an ElasticSearch query SearchSourceBuilder searchSourceBuilder = createSearchRequest(firstRow, maxRows, queryBuilder); - SearchResult result = executeSearch(table, searchSourceBuilder, false); + SearchResult result = executeSearch(table, searchSourceBuilder, scrollNeeded(maxRows)); return new JestElasticSearchDataSet(elasticSearchClient, result, selectItems); } return super.materializeMainSchemaTable(table, selectItems, whereItems, firstRow, maxRows); } + private boolean scrollNeeded(int maxRows) { + // if either we don't know about max rows or max rows is set higher than threshold + return !limitMaxRowsIsSet(maxRows) || maxRows > SCROLL_THRESHOLD; + } + private SearchResult executeSearch(Table table, SearchSourceBuilder searchSourceBuilder, boolean scroll) { Search.Builder builder = new Search.Builder(searchSourceBuilder.toString()).addIndex(getIndexName()).addType( table.getName()); @@ -292,7 +300,7 @@ public class ElasticSearchRestDataContext extends QueryPostprocessDataContext im @Override protected DataSet materializeMainSchemaTable(Table table, Column[] columns, int maxRows) { - SearchResult searchResult = executeSearch(table, createSearchRequest(1, maxRows, null), limitMaxRowsIsSet( + SearchResult searchResult = executeSearch(table, createSearchRequest(1, maxRows, null), scrollNeeded( maxRows)); return new JestElasticSearchDataSet(elasticSearchClient, searchResult, columns); @@ -306,6 +314,8 @@ public class ElasticSearchRestDataContext extends QueryPostprocessDataContext im } if (limitMaxRowsIsSet(maxRows)) { searchRequest.size(maxRows); + } else { + searchRequest.size(Integer.MAX_VALUE); } if (queryBuilder != null) {
