[5/5] git commit: PHOENIX-1251 Salted queries with range scan become full table scans

2014-10-03 Thread jamestaylor 
PHOENIX-1251 Salted queries with range scan become full table scans


Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/51f69bcb
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/51f69bcb
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/51f69bcb

Branch: refs/heads/4.0
Commit: 51f69bcb6f09fa3fd7e4194fc192bfd0bde6c60d
Parents: 7a16a08
Author: James Taylor jtay...@salesforce.com
Authored: Wed Oct 1 23:51:52 2014 -0700
Committer: James Taylor jtay...@salesforce.com
Committed: Fri Oct 3 16:31:28 2014 -0700

--
 .../BaseParallelIteratorsRegionSplitterIT.java  |  90 
 .../end2end/BaseTenantSpecificViewIndexIT.java  |   9 +-
 .../org/apache/phoenix/end2end/BaseViewIT.java  |   4 +-
 ...efaultParallelIteratorsRegionSplitterIT.java | 163 --
 .../org/apache/phoenix/end2end/DeleteIT.java|   1 +
 .../phoenix/end2end/GuidePostsLifeCycleIT.java  | 168 --
 .../org/apache/phoenix/end2end/InListIT.java|   7 +-
 .../org/apache/phoenix/end2end/KeyOnlyIT.java   |  57 +
 .../phoenix/end2end/MultiCfQueryExecIT.java |  73 +--
 .../phoenix/end2end/ParallelIteratorsIT.java| 172 +++
 .../org/apache/phoenix/end2end/QueryPlanIT.java | 202 -
 ...ipRangeParallelIteratorRegionSplitterIT.java | 109 --
 .../end2end/SkipScanAfterManualSplitIT.java |   6 +-
 .../apache/phoenix/end2end/StatsManagerIT.java  | 198 -
 .../end2end/TenantSpecificTablesDMLIT.java  |  55 +
 .../phoenix/end2end/index/LocalIndexIT.java |   2 +
 .../phoenix/end2end/index/MutableIndexIT.java   |   1 +
 .../phoenix/end2end/index/SaltedIndexIT.java|   4 +-
 .../apache/phoenix/cache/ServerCacheClient.java |  15 +-
 .../org/apache/phoenix/compile/QueryPlan.java   |   2 +
 .../org/apache/phoenix/compile/ScanRanges.java  | 207 +++---
 .../phoenix/compile/StatementContext.java   |  41 +---
 .../apache/phoenix/compile/WhereOptimizer.java  |   6 +-
 .../coprocessor/MetaDataEndpointImpl.java   |  82 +++
 .../apache/phoenix/execute/AggregatePlan.java   |   2 +-
 .../apache/phoenix/execute/BaseQueryPlan.java   |   5 +
 .../apache/phoenix/execute/HashJoinPlan.java|   5 +
 .../org/apache/phoenix/execute/ScanPlan.java|   4 +-
 .../apache/phoenix/filter/SkipScanFilter.java   |  34 ++-
 .../DefaultParallelIteratorRegionSplitter.java  | 173 ---
 .../apache/phoenix/iterate/ExplainTable.java|   9 +-
 ...ocalIndexParallelIteratorRegionSplitter.java |  43 
 .../ParallelIteratorRegionSplitterFactory.java  |  42 
 .../phoenix/iterate/ParallelIterators.java  | 173 +--
 ...SkipRangeParallelIteratorRegionSplitter.java |  83 ---
 .../apache/phoenix/jdbc/PhoenixStatement.java   |   5 +
 .../java/org/apache/phoenix/query/KeyRange.java |   8 +
 .../org/apache/phoenix/query/StatsManager.java  |  59 -
 .../apache/phoenix/query/StatsManagerImpl.java  | 218 ---
 .../schema/stat/StatisticsCollector.java|  48 ++--
 .../phoenix/schema/stat/StatisticsTable.java|   6 +-
 .../java/org/apache/phoenix/util/ScanUtil.java  |  65 +-
 .../compile/ScanRangesIntersectTest.java| 105 +
 .../apache/phoenix/compile/ScanRangesTest.java  |   2 +-
 .../phoenix/compile/WhereCompilerTest.java  |  13 +-
 .../phoenix/compile/WhereOptimizerTest.java |   5 +-
 .../query/BaseConnectionlessQueryTest.java  |   3 +-
 .../org/apache/phoenix/query/QueryPlanTest.java | 179 +++
 .../java/org/apache/phoenix/util/TestUtil.java  |  41 
 .../phoenix/pig/hadoop/PhoenixInputFormat.java  |  25 +--
 50 files changed, 1079 insertions(+), 1950 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/phoenix/blob/51f69bcb/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseParallelIteratorsRegionSplitterIT.java
--
diff --git 
a/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseParallelIteratorsRegionSplitterIT.java
 
b/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseParallelIteratorsRegionSplitterIT.java
deleted file mode 100644
index 514b36e..000
--- 
a/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseParallelIteratorsRegionSplitterIT.java
+++ /dev/null
@@ -1,90 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * License); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- *

[4/5] PHOENIX-1251 Salted queries with range scan become full table scans

2014-10-03 Thread jamestaylor 
http://git-wip-us.apache.org/repos/asf/phoenix/blob/51f69bcb/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/LocalIndexIT.java
--
diff --git 
a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/LocalIndexIT.java 
b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/LocalIndexIT.java
index 21fb970..376590a 100644
--- 
a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/LocalIndexIT.java
+++ 
b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/LocalIndexIT.java
@@ -59,6 +59,7 @@ import org.apache.phoenix.util.QueryUtil;
 import org.apache.phoenix.util.ReadOnlyProps;
 import org.apache.phoenix.util.TestUtil;
 import org.junit.BeforeClass;
+import org.junit.Ignore;
 import org.junit.Test;
 
 import com.google.common.collect.Maps;
@@ -632,6 +633,7 @@ public class LocalIndexIT extends BaseIndexIT {
 }
 
 @Test
+@Ignore // TODO: ask Rajeshbabu to take a look
 public void testLocalIndexScanAfterRegionSplit() throws Exception {
 createBaseTable(DATA_TABLE_NAME, null, ('e','j','o'));
 Connection conn1 = DriverManager.getConnection(getUrl());

http://git-wip-us.apache.org/repos/asf/phoenix/blob/51f69bcb/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexIT.java
--
diff --git 
a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexIT.java 
b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexIT.java
index fe24c35..b093acb 100644
--- 
a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexIT.java
+++ 
b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexIT.java
@@ -191,6 +191,7 @@ public class MutableIndexIT extends BaseMutableIndexIT {
 }
 
 @Test
+//@Ignore // TODO: ask Rajeshbabu to look at: SkipScanFilter:151 assert 
for skip_hint  current_key is failing 
 public void testCoveredColumnUpdatesWithLocalIndex() throws Exception {
 testCoveredColumnUpdates(true);
 }

http://git-wip-us.apache.org/repos/asf/phoenix/blob/51f69bcb/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/SaltedIndexIT.java
--
diff --git 
a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/SaltedIndexIT.java 
b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/SaltedIndexIT.java
index d5e9d42..8f7912a 100644
--- 
a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/SaltedIndexIT.java
+++ 
b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/SaltedIndexIT.java
@@ -145,7 +145,7 @@ public class SaltedIndexIT extends BaseIndexIT {
 rs = conn.createStatement().executeQuery(EXPLAIN  + query);
 expectedPlan = indexSaltBuckets == null ? 
  CLIENT PARALLEL 1-WAY RANGE SCAN OVER  + INDEX_TABLE_FULL_NAME 
+  [~'y'] : 
-(CLIENT PARALLEL 4-WAY SKIP SCAN ON 4 KEYS OVER  + 
INDEX_TABLE_FULL_NAME +  [0,~'y'] - [3,~'y']\n + 
+(CLIENT PARALLEL 4-WAY RANGE SCAN OVER  + INDEX_TABLE_FULL_NAME 
+  [0,~'y']\n + 
  CLIENT MERGE SORT);
 assertEquals(expectedPlan,QueryUtil.getExplainPlan(rs));
 
@@ -164,7 +164,7 @@ public class SaltedIndexIT extends BaseIndexIT {
 rs = conn.createStatement().executeQuery(EXPLAIN  + query);
 expectedPlan = indexSaltBuckets == null ? 
 CLIENT PARALLEL 1-WAY RANGE SCAN OVER  + INDEX_TABLE_FULL_NAME + 
 [*] - [~'x'] :
-(CLIENT PARALLEL 4-WAY SKIP SCAN ON 4 RANGES OVER  + 
INDEX_TABLE_FULL_NAME +  [0,*] - [3,~'x']\n + 
+(CLIENT PARALLEL 4-WAY RANGE SCAN OVER  + INDEX_TABLE_FULL_NAME 
+  [0,*] - [0,~'x']\n + 
  CLIENT MERGE SORT);
 assertEquals(expectedPlan,QueryUtil.getExplainPlan(rs));
 

http://git-wip-us.apache.org/repos/asf/phoenix/blob/51f69bcb/phoenix-core/src/main/java/org/apache/phoenix/cache/ServerCacheClient.java
--
diff --git 
a/phoenix-core/src/main/java/org/apache/phoenix/cache/ServerCacheClient.java 
b/phoenix-core/src/main/java/org/apache/phoenix/cache/ServerCacheClient.java
index fa19881..f22f874 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/cache/ServerCacheClient.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/cache/ServerCacheClient.java
@@ -17,7 +17,6 @@
  */
 package org.apache.phoenix.cache;
 
-import static java.util.Collections.emptyMap;
 import static org.apache.phoenix.util.LogUtil.addCustomAnnotations;
 
 import java.io.Closeable;
@@ -60,10 +59,13 @@ import org.apache.phoenix.memory.MemoryManager.MemoryChunk;
 import org.apache.phoenix.query.ConnectionQueryServices;
 import org.apache.phoenix.query.QueryServices;
 import org.apache.phoenix.query.QueryServicesOptions;
+import org.apache.phoenix.schema.PTable;
+import

[1/5] git commit: PHOENIX-1251 Salted queries with range scan become full table scans

2014-10-03 Thread jamestaylor 
Repository: phoenix
Updated Branches:
  refs/heads/4.0 f7e6a6c7e - 51f69bcb6


PHOENIX-1251 Salted queries with range scan become full table scans


Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/a5d07cc0
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/a5d07cc0
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/a5d07cc0

Branch: refs/heads/4.0
Commit: a5d07cc076763000f0c48c4c958e33aa578e85a9
Parents: 846ed10
Author: James Taylor jtay...@salesforce.com
Authored: Wed Oct 1 08:49:04 2014 -0700
Committer: James Taylor jtay...@salesforce.com
Committed: Wed Oct 1 08:49:04 2014 -0700

--
 .../org/apache/phoenix/end2end/InListIT.java|   2 +-
 .../org/apache/phoenix/compile/ScanRanges.java  | 239 ++--
 .../phoenix/compile/StatementContext.java   |  21 +-
 .../apache/phoenix/compile/WhereOptimizer.java  |  33 +-
 .../phoenix/index/PhoenixIndexBuilder.java  |   1 -
 .../phoenix/iterate/ParallelIterators.java  | 391 ++-
 .../java/org/apache/phoenix/util/ScanUtil.java  |   4 +-
 7 files changed, 530 insertions(+), 161 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/phoenix/blob/a5d07cc0/phoenix-core/src/it/java/org/apache/phoenix/end2end/InListIT.java
--
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/InListIT.java 
b/phoenix-core/src/it/java/org/apache/phoenix/end2end/InListIT.java
index dc60b69..60bcb65 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/InListIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/InListIT.java
@@ -163,7 +163,7 @@ public class InListIT extends BaseHBaseManagedTimeIT {
 // the different combinations to check each test against
 private static final ListBoolean TENANCIES = Arrays.asList(false, true);
 private static final ListPDataType INTEGER_TYPES = 
Arrays.asList(PDataType.INTEGER, PDataType.LONG);
-private static final ListInteger SALT_BUCKET_NUMBERS = Arrays.asList(0, 
4);
+private static final ListInteger SALT_BUCKET_NUMBERS = 
Arrays.asList(/*0,*/ 4);
 
 // we should be including the RANGE_SCAN hint here, but a bug with 
ParallelIterators causes tests to fail
 // see the relevant JIRA here: 
https://issues.apache.org/jira/browse/PHOENIX-1251

http://git-wip-us.apache.org/repos/asf/phoenix/blob/a5d07cc0/phoenix-core/src/main/java/org/apache/phoenix/compile/ScanRanges.java
--
diff --git 
a/phoenix-core/src/main/java/org/apache/phoenix/compile/ScanRanges.java 
b/phoenix-core/src/main/java/org/apache/phoenix/compile/ScanRanges.java
index dc8e0b3..1c739f3 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/compile/ScanRanges.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/ScanRanges.java
@@ -23,12 +23,17 @@ import java.util.Iterator;
 import java.util.List;
 
 import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.filter.Filter;
+import org.apache.hadoop.hbase.filter.FilterList;
 import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
+import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.phoenix.filter.SkipScanFilter;
 import org.apache.phoenix.query.KeyRange;
 import org.apache.phoenix.query.KeyRange.Bound;
+import org.apache.phoenix.query.QueryConstants;
 import org.apache.phoenix.schema.RowKeySchema;
 import org.apache.phoenix.schema.SaltingUtil;
+import org.apache.phoenix.util.ByteUtil;
 import org.apache.phoenix.util.ScanUtil;
 import org.apache.phoenix.util.SchemaUtil;
 
@@ -40,8 +45,8 @@ import com.google.common.collect.Lists;
 public class ScanRanges {
 private static final ListListKeyRange EVERYTHING_RANGES = 
Collections.ListKeyRangeemptyList();
 private static final ListListKeyRange NOTHING_RANGES = 
Collections.ListKeyRangesingletonList(Collections.KeyRangesingletonList(KeyRange.EMPTY_RANGE));
-public static final ScanRanges EVERYTHING = new 
ScanRanges(null,ScanUtil.SINGLE_COLUMN_SLOT_SPAN,EVERYTHING_RANGES, false, 
false);
-public static final ScanRanges NOTHING = new 
ScanRanges(null,ScanUtil.SINGLE_COLUMN_SLOT_SPAN,NOTHING_RANGES, false, false);
+public static final ScanRanges EVERYTHING = new 
ScanRanges(null,ScanUtil.SINGLE_COLUMN_SLOT_SPAN,EVERYTHING_RANGES, 
KeyRange.EVERYTHING_RANGE, false, false, null);
+public static final ScanRanges NOTHING = new 
ScanRanges(null,ScanUtil.SINGLE_COLUMN_SLOT_SPAN,NOTHING_RANGES, 
KeyRange.EMPTY_RANGE, false, false, null);
 
 public static ScanRanges create(RowKeySchema schema, ListListKeyRange 
ranges, int[] slotSpan) {
 return create(schema, ranges, slotSpan, false, null);
@@ -72,37 +77,210 @@ public class ScanRanges {
 // when there's a single

[3/5] PHOENIX-1251 Salted queries with range scan become full table scans

2014-10-03 Thread jamestaylor 
http://git-wip-us.apache.org/repos/asf/phoenix/blob/51f69bcb/phoenix-core/src/main/java/org/apache/phoenix/iterate/SkipRangeParallelIteratorRegionSplitter.java
--
diff --git 
a/phoenix-core/src/main/java/org/apache/phoenix/iterate/SkipRangeParallelIteratorRegionSplitter.java
 
b/phoenix-core/src/main/java/org/apache/phoenix/iterate/SkipRangeParallelIteratorRegionSplitter.java
deleted file mode 100644
index 81f5af6..000
--- 
a/phoenix-core/src/main/java/org/apache/phoenix/iterate/SkipRangeParallelIteratorRegionSplitter.java
+++ /dev/null
@@ -1,83 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * License); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an AS IS BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.phoenix.iterate;
-
-import java.sql.SQLException;
-import java.util.List;
-
-import org.apache.hadoop.hbase.HRegionLocation;
-import org.apache.phoenix.compile.ScanRanges;
-import org.apache.phoenix.compile.StatementContext;
-import org.apache.phoenix.parse.HintNode;
-import org.apache.phoenix.query.KeyRange;
-import org.apache.phoenix.schema.PTable;
-import org.apache.phoenix.schema.SaltingUtil;
-
-import com.google.common.base.Predicate;
-import com.google.common.collect.Iterables;
-import com.google.common.collect.Lists;
-
-
-/**
- * Split the region according to the information contained in the scan's 
SkipScanFilter.
- */
-public class SkipRangeParallelIteratorRegionSplitter extends 
DefaultParallelIteratorRegionSplitter {
-
-public static SkipRangeParallelIteratorRegionSplitter 
getInstance(StatementContext context, PTable table, HintNode hintNode) {
-return new SkipRangeParallelIteratorRegionSplitter(context, table, 
hintNode);
-}
-
-protected SkipRangeParallelIteratorRegionSplitter(StatementContext 
context, PTable table, HintNode hintNode) {
-super(context, table, hintNode);
-}
-
-@Override
-protected ListHRegionLocation getAllRegions() throws SQLException {
-ListHRegionLocation allTableRegions = 
context.getConnection().getQueryServices().getAllTableRegions(table.getPhysicalName().getBytes());
-return filterRegions(allTableRegions, context.getScanRanges());
-}
-
-public ListHRegionLocation filterRegions(ListHRegionLocation 
allTableRegions, final ScanRanges ranges) {
-IterableHRegionLocation regions;
-if (ranges == ScanRanges.EVERYTHING) {
-return allTableRegions;
-} else if (ranges == ScanRanges.NOTHING) { // TODO: why not emptyList?
-return Lists.HRegionLocationnewArrayList();
-} else {
-regions = Iterables.filter(allTableRegions,
-new PredicateHRegionLocation() {
-@Override
-public boolean apply(HRegionLocation region) {
-KeyRange minMaxRange = context.getMinMaxRange();
-if (minMaxRange != null) {
-KeyRange range = 
KeyRange.getKeyRange(region.getRegionInfo().getStartKey(), 
region.getRegionInfo().getEndKey());
-if (table.getBucketNum() != null) {
-// Add salt byte, as minMaxRange won't have it
-minMaxRange = 
SaltingUtil.addSaltByte(region.getRegionInfo().getStartKey(), minMaxRange);
-}
-range = range.intersect(minMaxRange);
-return ranges.intersect(range.getLowerRange(), 
range.getUpperRange());
-}
-return 
ranges.intersect(region.getRegionInfo().getStartKey(), 
region.getRegionInfo().getEndKey());
-}
-});
-}
-return Lists.newArrayList(regions);
-}
-
-}

http://git-wip-us.apache.org/repos/asf/phoenix/blob/51f69bcb/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixStatement.java
--
diff --git 
a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixStatement.java 
b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixStatement.java
index f7d6e14..4f67d4f 100644
---

git commit: PHOENIX-1310: Spill files filling up /tmp on server(Siddharth Wagle)

2014-10-03 Thread jeffreyz 
Repository: phoenix
Updated Branches:
  refs/heads/master a55c03cc9 - 051e40843


PHOENIX-1310: Spill files filling up /tmp on server(Siddharth Wagle)


Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/051e4084
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/051e4084
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/051e4084

Branch: refs/heads/master
Commit: 051e40843bb2c8154d31f54ef17ae66cd2824d4f
Parents: a55c03c
Author: Jeffrey Zhong jeffr...@apache.org
Authored: Fri Oct 3 17:18:01 2014 -0700
Committer: Jeffrey Zhong jeffr...@apache.org
Committed: Fri Oct 3 17:18:01 2014 -0700

--
 .../phoenix/cache/aggcache/SpillFile.java   | 42 +++-
 .../phoenix/cache/aggcache/SpillManager.java|  6 ++-
 2 files changed, 28 insertions(+), 20 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/phoenix/blob/051e4084/phoenix-core/src/main/java/org/apache/phoenix/cache/aggcache/SpillFile.java
--
diff --git 
a/phoenix-core/src/main/java/org/apache/phoenix/cache/aggcache/SpillFile.java 
b/phoenix-core/src/main/java/org/apache/phoenix/cache/aggcache/SpillFile.java
index 8dd64d0..51aef98 100644
--- 
a/phoenix-core/src/main/java/org/apache/phoenix/cache/aggcache/SpillFile.java
+++ 
b/phoenix-core/src/main/java/org/apache/phoenix/cache/aggcache/SpillFile.java
@@ -18,6 +18,12 @@
 
 package org.apache.phoenix.cache.aggcache;
 
+import com.google.common.collect.Maps;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.phoenix.util.Closeables;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
 import java.io.Closeable;
 import java.io.File;
 import java.io.IOException;
@@ -28,12 +34,6 @@ import java.nio.channels.FileChannel.MapMode;
 import java.util.Map;
 import java.util.UUID;
 
-import org.apache.phoenix.util.Closeables;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.collect.Maps;
-
 /**
  * This class abstracts a SpillFile It is a accessible on a per page basis
  * For every SpillFile object a single spill file is always created. 
@@ -49,6 +49,8 @@ public class SpillFile implements Closeable {
 static final int DEFAULT_PAGE_SIZE = 4096;
 // Map of initial SpillFile at index 0, and overflow spillFiles
 private MapInteger, TempFile tempFiles;
+// Custom spill files directory
+private File spillFilesDirectory = null;
 
 // Wrapper class for a TempFile: File + RandomAccessFile
 private static class TempFile implements Closeable{
@@ -81,22 +83,31 @@ public class SpillFile implements Closeable {
 }
}
 }
-
+
+private SpillFile(File spillFilesDirectory) throws IOException {
+  this.spillFilesDirectory = spillFilesDirectory;
+  this.tempFiles = Maps.newHashMap();
+  // Init the first pre-allocated spillFile
+  tempFiles.put(0, createTempFile());
+}
+
 /**
  * Create a new SpillFile using the Java TempFile creation function. 
SpillFile is access in
  * pages.
  */
-public static SpillFile createSpillFile() {
-   try {   
-   return new SpillFile(createTempFile()); 
+public static SpillFile createSpillFile(File spillFilesDir) {
+   try {
+   return new SpillFile(spillFilesDir);
} catch (IOException ioe) {
throw new RuntimeException(Could not create Spillfile  + ioe);
 }
 }
 
 
-private static TempFile createTempFile() throws IOException {
-File tempFile = File.createTempFile(UUID.randomUUID().toString(), 
null);
+private TempFile createTempFile() throws IOException {
+// Create temp file in temp dir or custom dir if provided
+File tempFile = File.createTempFile(UUID.randomUUID().toString(),
+  null, spillFilesDirectory);
 if (logger.isDebugEnabled()) {
 logger.debug(Creating new SpillFile:  + 
tempFile.getAbsolutePath());
 }
@@ -106,13 +117,6 @@ public class SpillFile implements Closeable {
 return new TempFile(tempFile, file);
 }
 
-
-private SpillFile(TempFile spFile) throws IOException {
-this.tempFiles = Maps.newHashMap();
-// Init the first pre-allocated spillFile
-tempFiles.put(0, spFile);
-}
-
 /**
  * Random access to a page of the current spill file
  * @param index

http://git-wip-us.apache.org/repos/asf/phoenix/blob/051e4084/phoenix-core/src/main/java/org/apache/phoenix/cache/aggcache/SpillManager.java
--
diff --git 
a/phoenix-core/src/main/java/org/apache/phoenix/cache/aggcache/SpillManager.java

[5/5] git commit: Merge branch 'master' of https://git-wip-us.apache.org/repos/asf/phoenix

2014-10-03 Thread jamestaylor 
Merge branch 'master' of https://git-wip-us.apache.org/repos/asf/phoenix


Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/cd8c6b87
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/cd8c6b87
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/cd8c6b87

Branch: refs/heads/master
Commit: cd8c6b877bca5b7405870d6688c3c8054e2d17f3
Parents: d018cc1 051e408
Author: James Taylor jtay...@salesforce.com
Authored: Fri Oct 3 17:33:33 2014 -0700
Committer: James Taylor jtay...@salesforce.com
Committed: Fri Oct 3 17:33:33 2014 -0700

--
 .../phoenix/cache/aggcache/SpillFile.java   | 42 +++-
 .../phoenix/cache/aggcache/SpillManager.java|  6 ++-
 2 files changed, 28 insertions(+), 20 deletions(-)
--

[4/5] git commit: PHOENIX-1251 Salted queries with range scan become full table scans

2014-10-03 Thread jamestaylor 
PHOENIX-1251 Salted queries with range scan become full table scans

Conflicts:

phoenix-core/src/it/java/org/apache/phoenix/end2end/SkipScanAfterManualSplitIT.java

phoenix-core/src/main/java/org/apache/phoenix/iterate/ParallelIterators.java


Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/d018cc1c
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/d018cc1c
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/d018cc1c

Branch: refs/heads/master
Commit: d018cc1c6e01d9836de6e67af4f8b91de3269bfd
Parents: a55c03c
Author: James Taylor jtay...@salesforce.com
Authored: Wed Oct 1 08:49:04 2014 -0700
Committer: James Taylor jtay...@salesforce.com
Committed: Fri Oct 3 17:30:42 2014 -0700

--
 .../BaseParallelIteratorsRegionSplitterIT.java  |  90 
 .../end2end/BaseTenantSpecificViewIndexIT.java  |   9 +-
 .../org/apache/phoenix/end2end/BaseViewIT.java  |   4 +-
 ...efaultParallelIteratorsRegionSplitterIT.java | 163 ---
 .../org/apache/phoenix/end2end/DeleteIT.java|   1 +
 .../phoenix/end2end/GuidePostsLifeCycleIT.java  | 168 ---
 .../org/apache/phoenix/end2end/InListIT.java|   7 +-
 .../org/apache/phoenix/end2end/KeyOnlyIT.java   |  57 +--
 .../phoenix/end2end/MultiCfQueryExecIT.java |  73 +--
 .../phoenix/end2end/ParallelIteratorsIT.java| 172 +++
 .../org/apache/phoenix/end2end/QueryPlanIT.java | 202 
 ...ipRangeParallelIteratorRegionSplitterIT.java | 109 -
 .../end2end/SkipScanAfterManualSplitIT.java |  30 +-
 .../apache/phoenix/end2end/StatsManagerIT.java  | 198 
 .../end2end/TenantSpecificTablesDMLIT.java  |  55 +--
 .../phoenix/end2end/index/LocalIndexIT.java |   2 +
 .../phoenix/end2end/index/MutableIndexIT.java   |   1 +
 .../phoenix/end2end/index/SaltedIndexIT.java|   4 +-
 .../apache/phoenix/cache/ServerCacheClient.java |  15 +-
 .../org/apache/phoenix/compile/QueryPlan.java   |   2 +
 .../org/apache/phoenix/compile/ScanRanges.java  | 370 ---
 .../phoenix/compile/StatementContext.java   |  46 +-
 .../apache/phoenix/compile/WhereOptimizer.java  |  39 +-
 .../coprocessor/MetaDataEndpointImpl.java   |  82 ++--
 .../apache/phoenix/execute/AggregatePlan.java   |   2 +-
 .../apache/phoenix/execute/BaseQueryPlan.java   |   5 +
 .../apache/phoenix/execute/HashJoinPlan.java|   5 +
 .../org/apache/phoenix/execute/ScanPlan.java|   4 +-
 .../apache/phoenix/filter/SkipScanFilter.java   |  34 +-
 .../phoenix/index/PhoenixIndexBuilder.java  |   1 -
 .../DefaultParallelIteratorRegionSplitter.java  | 173 ---
 .../apache/phoenix/iterate/ExplainTable.java|   9 +-
 ...ocalIndexParallelIteratorRegionSplitter.java |  43 --
 .../ParallelIteratorRegionSplitterFactory.java  |  42 --
 .../phoenix/iterate/ParallelIterators.java  | 472 ++-
 ...SkipRangeParallelIteratorRegionSplitter.java |  83 
 .../apache/phoenix/jdbc/PhoenixStatement.java   |   5 +
 .../java/org/apache/phoenix/query/KeyRange.java |   8 +
 .../org/apache/phoenix/query/StatsManager.java  |  59 ---
 .../apache/phoenix/query/StatsManagerImpl.java  | 218 -
 .../schema/stat/StatisticsCollector.java|  48 +-
 .../phoenix/schema/stat/StatisticsTable.java|   6 +-
 .../java/org/apache/phoenix/util/ScanUtil.java  |  69 ++-
 .../compile/ScanRangesIntersectTest.java| 105 +
 .../apache/phoenix/compile/ScanRangesTest.java  |   2 +-
 .../phoenix/compile/WhereCompilerTest.java  |  13 +-
 .../phoenix/compile/WhereOptimizerTest.java |   5 +-
 .../query/BaseConnectionlessQueryTest.java  |   3 +-
 .../org/apache/phoenix/query/QueryPlanTest.java | 179 +++
 .../java/org/apache/phoenix/util/TestUtil.java  |  41 ++
 .../phoenix/pig/hadoop/PhoenixInputFormat.java  |  25 +-
 51 files changed, 1540 insertions(+), 2018 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/phoenix/blob/d018cc1c/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseParallelIteratorsRegionSplitterIT.java
--
diff --git 
a/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseParallelIteratorsRegionSplitterIT.java
 
b/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseParallelIteratorsRegionSplitterIT.java
deleted file mode 100644
index 514b36e..000
--- 
a/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseParallelIteratorsRegionSplitterIT.java
+++ /dev/null
@@ -1,90 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * License); you may not use this file

[3/5] PHOENIX-1251 Salted queries with range scan become full table scans

2014-10-03 Thread jamestaylor 
http://git-wip-us.apache.org/repos/asf/phoenix/blob/d018cc1c/phoenix-core/src/it/java/org/apache/phoenix/end2end/TenantSpecificTablesDMLIT.java
--
diff --git 
a/phoenix-core/src/it/java/org/apache/phoenix/end2end/TenantSpecificTablesDMLIT.java
 
b/phoenix-core/src/it/java/org/apache/phoenix/end2end/TenantSpecificTablesDMLIT.java
index e4c17f9..b01c0ab 100644
--- 
a/phoenix-core/src/it/java/org/apache/phoenix/end2end/TenantSpecificTablesDMLIT.java
+++ 
b/phoenix-core/src/it/java/org/apache/phoenix/end2end/TenantSpecificTablesDMLIT.java
@@ -18,34 +18,22 @@
 package org.apache.phoenix.end2end;
 
 import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES;
+import static org.apache.phoenix.util.TestUtil.analyzeTable;
+import static org.apache.phoenix.util.TestUtil.getAllSplits;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
-import java.io.IOException;
 import java.sql.Connection;
 import java.sql.DriverManager;
 import java.sql.ResultSet;
 import java.sql.SQLException;
-import java.util.Collections;
-import java.util.Comparator;
 import java.util.List;
 import java.util.Properties;
 
-import org.apache.hadoop.hbase.HRegionLocation;
-import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.phoenix.compile.SequenceManager;
-import org.apache.phoenix.compile.StatementContext;
-import org.apache.phoenix.iterate.DefaultParallelIteratorRegionSplitter;
-import org.apache.phoenix.jdbc.PhoenixConnection;
-import org.apache.phoenix.jdbc.PhoenixStatement;
-import org.apache.phoenix.parse.HintNode;
 import org.apache.phoenix.query.KeyRange;
-import org.apache.phoenix.schema.PTableKey;
 import org.apache.phoenix.schema.TableNotFoundException;
-import org.apache.phoenix.schema.TableRef;
 import org.apache.phoenix.util.PhoenixRuntime;
 import org.apache.phoenix.util.PropertiesUtil;
 import org.junit.Test;
@@ -155,9 +143,8 @@ public class TenantSpecificTablesDMLIT extends 
BaseTenantSpecificTablesIT {
 assertTrue(Expected 1 row in result set, rs.next());
 assertEquals(2, rs.getInt(3));
 assertEquals(Viva Las Vegas, rs.getString(4));
-conn1 = nextConnection(getUrl());
-ListKeyRange splits = getSplits(conn1, new Scan());
-assertEquals(splits.size(), 5);
+ListKeyRange splits = getAllSplits(conn1, TENANT_TABLE_NAME);
+assertEquals(3, splits.size());
 }
 finally {
 conn1.close();
@@ -493,10 +480,6 @@ public class TenantSpecificTablesDMLIT extends 
BaseTenantSpecificTablesIT {
 }
 }
 
-private void analyzeTable(Connection conn, String tableName) throws 
IOException, SQLException {
-String query = ANALYZE  + tableName;
-conn.createStatement().execute(query);
-}
 @Test
 public void testUpsertValuesUsingViewWithNoWhereClause() throws Exception {
 Connection conn = nextConnection(PHOENIX_JDBC_TENANT_SPECIFIC_URL);
@@ -511,34 +494,4 @@ public class TenantSpecificTablesDMLIT extends 
BaseTenantSpecificTablesIT {
 assertFalse(rs.next());
 conn.close();
 }
-private static ListKeyRange getSplits(Connection conn, final Scan scan) 
throws SQLException {
-TableRef tableRef = getTableRef(conn);
-PhoenixConnection pconn = conn.unwrap(PhoenixConnection.class);
-final ListHRegionLocation regions = 
pconn.getQueryServices().getAllTableRegions(
-tableRef.getTable().getPhysicalName().getBytes());
-PhoenixStatement statement = new PhoenixStatement(pconn);
-StatementContext context = new StatementContext(statement, null, scan, 
new SequenceManager(statement));
-DefaultParallelIteratorRegionSplitter splitter = new 
DefaultParallelIteratorRegionSplitter(context, tableRef.getTable(),
-HintNode.EMPTY_HINT_NODE) {
-@Override
-protected ListHRegionLocation getAllRegions() throws 
SQLException {
-return 
DefaultParallelIteratorRegionSplitter.filterRegions(regions, scan.getStartRow(),
-scan.getStopRow());
-}
-};
-ListKeyRange keyRanges = splitter.getSplits();
-Collections.sort(keyRanges, new ComparatorKeyRange() {
-@Override
-public int compare(KeyRange o1, KeyRange o2) {
-return Bytes.compareTo(o1.getLowerRange(), o2.getLowerRange());
-}
-});
-return keyRanges;
-}
-protected static TableRef getTableRef(Connection conn) throws SQLException 
{
-PhoenixConnection pconn = conn.unwrap(PhoenixConnection.class);
-TableRef table = new TableRef(null, pconn.getMetaDataCache().getTable(
-new PTableKey(pconn.getTenantId(),

Apache-Phoenix | 4.0 | Hadoop1 | Build Successful

2014-10-03 Thread Apache Jenkins Server 
4.0 branch build status Successful

Source repository https://git-wip-us.apache.org/repos/asf/incubator-phoenix.git

Compiled Artifacts https://builds.apache.org/job/Phoenix-4.0-hadoop1/lastSuccessfulBuild/artifact/

Test Report https://builds.apache.org/job/Phoenix-4.0-hadoop1/lastCompletedBuild/testReport/

Changes
[jtaylor] PHOENIX-1251 Salted queries with range scan become full table scans

[jtaylor] PHOENIX-1251 Salted queries with range scan become full table scans

git commit: PHOENIX-1310: Spill files filling up /tmp on server(Siddharth Wagle)

2014-10-03 Thread jeffreyz 
Repository: phoenix
Updated Branches:
  refs/heads/3.0 ff47a9594 - d3e6a9fa2


PHOENIX-1310: Spill files filling up /tmp on server(Siddharth Wagle)


Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/d3e6a9fa
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/d3e6a9fa
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/d3e6a9fa

Branch: refs/heads/3.0
Commit: d3e6a9fa2e6c660ffd0eca9f99eae391070bb6c3
Parents: ff47a95
Author: Jeffrey Zhong jeffr...@apache.org
Authored: Fri Oct 3 17:18:01 2014 -0700
Committer: Jeffrey Zhong jeffr...@apache.org
Committed: Fri Oct 3 17:40:12 2014 -0700

--
 .../phoenix/cache/aggcache/SpillFile.java   | 42 +++-
 .../phoenix/cache/aggcache/SpillManager.java|  6 ++-
 2 files changed, 28 insertions(+), 20 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/phoenix/blob/d3e6a9fa/phoenix-core/src/main/java/org/apache/phoenix/cache/aggcache/SpillFile.java
--
diff --git 
a/phoenix-core/src/main/java/org/apache/phoenix/cache/aggcache/SpillFile.java 
b/phoenix-core/src/main/java/org/apache/phoenix/cache/aggcache/SpillFile.java
index 8dd64d0..51aef98 100644
--- 
a/phoenix-core/src/main/java/org/apache/phoenix/cache/aggcache/SpillFile.java
+++ 
b/phoenix-core/src/main/java/org/apache/phoenix/cache/aggcache/SpillFile.java
@@ -18,6 +18,12 @@
 
 package org.apache.phoenix.cache.aggcache;
 
+import com.google.common.collect.Maps;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.phoenix.util.Closeables;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
 import java.io.Closeable;
 import java.io.File;
 import java.io.IOException;
@@ -28,12 +34,6 @@ import java.nio.channels.FileChannel.MapMode;
 import java.util.Map;
 import java.util.UUID;
 
-import org.apache.phoenix.util.Closeables;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.collect.Maps;
-
 /**
  * This class abstracts a SpillFile It is a accessible on a per page basis
  * For every SpillFile object a single spill file is always created. 
@@ -49,6 +49,8 @@ public class SpillFile implements Closeable {
 static final int DEFAULT_PAGE_SIZE = 4096;
 // Map of initial SpillFile at index 0, and overflow spillFiles
 private MapInteger, TempFile tempFiles;
+// Custom spill files directory
+private File spillFilesDirectory = null;
 
 // Wrapper class for a TempFile: File + RandomAccessFile
 private static class TempFile implements Closeable{
@@ -81,22 +83,31 @@ public class SpillFile implements Closeable {
 }
}
 }
-
+
+private SpillFile(File spillFilesDirectory) throws IOException {
+  this.spillFilesDirectory = spillFilesDirectory;
+  this.tempFiles = Maps.newHashMap();
+  // Init the first pre-allocated spillFile
+  tempFiles.put(0, createTempFile());
+}
+
 /**
  * Create a new SpillFile using the Java TempFile creation function. 
SpillFile is access in
  * pages.
  */
-public static SpillFile createSpillFile() {
-   try {   
-   return new SpillFile(createTempFile()); 
+public static SpillFile createSpillFile(File spillFilesDir) {
+   try {
+   return new SpillFile(spillFilesDir);
} catch (IOException ioe) {
throw new RuntimeException(Could not create Spillfile  + ioe);
 }
 }
 
 
-private static TempFile createTempFile() throws IOException {
-File tempFile = File.createTempFile(UUID.randomUUID().toString(), 
null);
+private TempFile createTempFile() throws IOException {
+// Create temp file in temp dir or custom dir if provided
+File tempFile = File.createTempFile(UUID.randomUUID().toString(),
+  null, spillFilesDirectory);
 if (logger.isDebugEnabled()) {
 logger.debug(Creating new SpillFile:  + 
tempFile.getAbsolutePath());
 }
@@ -106,13 +117,6 @@ public class SpillFile implements Closeable {
 return new TempFile(tempFile, file);
 }
 
-
-private SpillFile(TempFile spFile) throws IOException {
-this.tempFiles = Maps.newHashMap();
-// Init the first pre-allocated spillFile
-tempFiles.put(0, spFile);
-}
-
 /**
  * Random access to a page of the current spill file
  * @param index

http://git-wip-us.apache.org/repos/asf/phoenix/blob/d3e6a9fa/phoenix-core/src/main/java/org/apache/phoenix/cache/aggcache/SpillManager.java
--
diff --git 
a/phoenix-core/src/main/java/org/apache/phoenix/cache/aggcache/SpillManager.java

Build failed in Jenkins: Phoenix | Master | Hadoop1 #397

2014-10-03 Thread Apache Jenkins Server 
See https://builds.apache.org/job/Phoenix-master-hadoop1/397/changes

Changes:

[jeffreyz] PHOENIX-1310: Spill files filling up /tmp on server(Siddharth Wagle)

[jtaylor] PHOENIX-1251 Salted queries with range scan become full table scans

--
[...truncated 646 lines...]
at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:94)
at java.lang.Thread.run(Thread.java:724)
Caused by: org.apache.phoenix.memory.InsufficientMemoryException: Requested 
memory of 104000 bytes is larger than global pool of 4 bytes.
at 
org.apache.phoenix.memory.GlobalMemoryManager.allocateBytes(GlobalMemoryManager.java:72)
at 
org.apache.phoenix.memory.GlobalMemoryManager.allocate(GlobalMemoryManager.java:100)
at 
org.apache.phoenix.memory.GlobalMemoryManager.allocate(GlobalMemoryManager.java:106)
at 
org.apache.phoenix.coprocessor.ScanRegionObserver.getTopNScanner(ScanRegionObserver.java:235)
at 
org.apache.phoenix.coprocessor.ScanRegionObserver.doPostScannerOpen(ScanRegionObserver.java:222)
at 
org.apache.phoenix.coprocessor.BaseScannerRegionObserver.postScannerOpen(BaseScannerRegionObserver.java:140)
... 8 more

at java.util.concurrent.FutureTask$Sync.innerGet(FutureTask.java:262)
at java.util.concurrent.FutureTask.get(FutureTask.java:119)
at 
org.apache.phoenix.iterate.ParallelIterators.getIterators(ParallelIterators.java:505)
at 
org.apache.phoenix.iterate.MergeSortResultIterator.getIterators(MergeSortResultIterator.java:48)
at 
org.apache.phoenix.iterate.MergeSortResultIterator.minIterator(MergeSortResultIterator.java:63)
at 
org.apache.phoenix.iterate.MergeSortResultIterator.next(MergeSortResultIterator.java:90)
at 
org.apache.phoenix.iterate.MergeSortTopNResultIterator.next(MergeSortTopNResultIterator.java:87)
at 
org.apache.phoenix.jdbc.PhoenixResultSet.next(PhoenixResultSet.java:732)
at 
org.apache.phoenix.end2end.SortOrderFIT.runQuery(SortOrderFIT.java:396)
at 
org.apache.phoenix.end2end.SortOrderFIT.runQueryTest(SortOrderFIT.java:353)
at 
org.apache.phoenix.end2end.SortOrderFIT.queryDescDateWithExplicitOrderBy(SortOrderFIT.java:251)
Caused by: org.apache.phoenix.exception.PhoenixIOException: 
org.apache.hadoop.hbase.DoNotRetryIOException: 
DESCCOLUMNSORTORDERTEST,,1412383112981.04d4acec0f96b59aa7692d14ab8f1c0b.: 
Requested memory of 104000 bytes is larger than global pool of 4 bytes.
at 
org.apache.phoenix.util.ServerUtil.createIOException(ServerUtil.java:77)
at 
org.apache.phoenix.util.ServerUtil.throwIOException(ServerUtil.java:45)
at 
org.apache.phoenix.coprocessor.BaseScannerRegionObserver.postScannerOpen(BaseScannerRegionObserver.java:158)
at 
org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost.postScannerOpen(RegionCoprocessorHost.java:1845)
at 
org.apache.hadoop.hbase.regionserver.HRegionServer.scan(HRegionServer.java:3092)
at 
org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29497)
at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2027)
at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:98)
at 
org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:114)
at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:94)
at java.lang.Thread.run(Thread.java:724)
Caused by: org.apache.phoenix.memory.InsufficientMemoryException: Requested 
memory of 104000 bytes is larger than global pool of 4 bytes.
at 
org.apache.phoenix.memory.GlobalMemoryManager.allocateBytes(GlobalMemoryManager.java:72)
at 
org.apache.phoenix.memory.GlobalMemoryManager.allocate(GlobalMemoryManager.java:100)
at 
org.apache.phoenix.memory.GlobalMemoryManager.allocate(GlobalMemoryManager.java:106)
at 
org.apache.phoenix.coprocessor.ScanRegionObserver.getTopNScanner(ScanRegionObserver.java:235)
at 
org.apache.phoenix.coprocessor.ScanRegionObserver.doPostScannerOpen(ScanRegionObserver.java:222)
at 
org.apache.phoenix.coprocessor.BaseScannerRegionObserver.postScannerOpen(BaseScannerRegionObserver.java:140)
... 8 more

at 
org.apache.phoenix.util.ServerUtil.parseServerException(ServerUtil.java:101)
at 
org.apache.phoenix.iterate.TableResultIterator.init(TableResultIterator.java:57)
at 
org.apache.phoenix.iterate.ParallelIterators$3.call(ParallelIterators.java:604)
at 
org.apache.phoenix.iterate.ParallelIterators$3.call(ParallelIterators.java:599)
at java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:334)
at java.util.concurrent.FutureTask.run(FutureTask.java:166)
at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
at

Build failed in Jenkins: Phoenix | 3.0 | Hadoop1 #238

2014-10-03 Thread Apache Jenkins Server 
See https://builds.apache.org/job/Phoenix-3.0-hadoop1/238/changes

Changes:

[jeffreyz] PHOENIX-1310: Spill files filling up /tmp on server(Siddharth Wagle)

--
[...truncated 9964 lines...]
  ProductMetricsITBaseClientManagedTimeIT.dropTables:72 null
  UpsertValuesITBaseClientManagedTimeIT.dropTables:72 null
  SaltedTableITBaseClientManagedTimeIT.dropTables:72 null
  SaltedTableVarLengthRowKeyITBaseClientManagedTimeIT.dropTables:72 null
  ScanQueryITBaseClientManagedTimeIT.dropTables:72 null
  CreateTableITBaseClientManagedTimeIT.dropTables:72 null
  
DefaultParallelIteratorsRegionSplitterITBaseClientManagedTimeIT.dropTables:72 
null
  CompareDecimalToLongITBaseClientManagedTimeIT.dropTables:72 null
  ArrayITBaseClientManagedTimeIT.dropTables:72 null
  TruncateFunctionITBaseClientManagedTimeIT.dropTables:72 null
  MultiCfQueryExecITBaseClientManagedTimeIT.dropTables:72 null
  RowValueConstructorITBaseClientManagedTimeIT.dropTables:72 null
  StddevITBaseClientManagedTimeIT.dropTables:72 null
  NotQueryITBaseClientManagedTimeIT.dropTables:72 null
  PercentileITBaseClientManagedTimeIT.dropTables:72 null
  IsNullITBaseClientManagedTimeIT.dropTables:72 null
  DistinctCountITBaseClientManagedTimeIT.dropTables:72 null
  InMemoryOrderByITBaseClientManagedTimeIT.dropTables:72 null
  
SkipRangeParallelIteratorRegionSplitterITBaseClientManagedTimeIT.dropTables:72 
null
  StatsManagerITBaseClientManagedTimeIT.dropTables:72 null

Tests in error: 
  
OrderByITBaseClientManagedTimeIT.doSetup:63-BaseTest.setUpTestDriver:476-BaseTest.checkClusterInitialized:455-BaseTest.setUpTestCluster:469-BaseTest.initMiniCluster:519
 » Runtime
  
KeyOnlyIT.doSetup:69-BaseTest.setUpTestDriver:476-BaseTest.checkClusterInitialized:455-BaseTest.setUpTestCluster:469-BaseTest.initMiniCluster:519
 » Runtime
  
CaseStatementITBaseQueryIT.doSetup:84-BaseTest.setUpTestDriver:476-BaseTest.checkClusterInitialized:455-BaseTest.setUpTestCluster:469-BaseTest.initMiniCluster:519
 » Runtime
  
CustomEntityDataITBaseClientManagedTimeIT.doSetup:63-BaseTest.setUpTestDriver:476-BaseTest.checkClusterInitialized:455-BaseTest.setUpTestCluster:469-BaseTest.initMiniCluster:519
 » Runtime
  
ToCharFunctionITBaseClientManagedTimeIT.doSetup:63-BaseTest.setUpTestDriver:476-BaseTest.checkClusterInitialized:455-BaseTest.setUpTestCluster:469-BaseTest.initMiniCluster:519
 » Runtime
  
GroupByITBaseQueryIT.doSetup:84-BaseTest.setUpTestDriver:476-BaseTest.checkClusterInitialized:455-BaseTest.setUpTestCluster:469-BaseTest.initMiniCluster:519
 » Runtime
  
TenantSpecificTablesDDLITBaseTenantSpecificTablesIT.doSetup:88-BaseTest.setUpTestDriver:476-BaseTest.checkClusterInitialized:455-BaseTest.setUpTestCluster:469-BaseTest.initMiniCluster:519
 » Runtime
  
QueryDatabaseMetaDataITBaseClientManagedTimeIT.doSetup:63-BaseTest.setUpTestDriver:476-BaseTest.checkClusterInitialized:455-BaseTest.setUpTestCluster:469-BaseTest.initMiniCluster:519
 » Runtime
  
VariableLengthPKITBaseClientManagedTimeIT.doSetup:63-BaseTest.setUpTestDriver:476-BaseTest.checkClusterInitialized:455-BaseTest.setUpTestCluster:469-BaseTest.initMiniCluster:519
 » Runtime
  
DerivedTableITBaseClientManagedTimeIT.doSetup:63-BaseTest.setUpTestDriver:476-BaseTest.checkClusterInitialized:455-BaseTest.setUpTestCluster:469-BaseTest.initMiniCluster:519
 » Runtime
  
ReadIsolationLevelITBaseClientManagedTimeIT.doSetup:63-BaseTest.setUpTestDriver:476-BaseTest.checkClusterInitialized:455-BaseTest.setUpTestCluster:469-BaseTest.initMiniCluster:519
 » Runtime
  
GroupByCaseITBaseClientManagedTimeIT.doSetup:63-BaseTest.setUpTestDriver:476-BaseTest.checkClusterInitialized:455-BaseTest.setUpTestCluster:469-BaseTest.initMiniCluster:519
 » Runtime
  
TopNITBaseClientManagedTimeIT.doSetup:63-BaseTest.setUpTestDriver:476-BaseTest.checkClusterInitialized:455-BaseTest.setUpTestCluster:469-BaseTest.initMiniCluster:519
 » Runtime
  
DynamicColumnITBaseClientManagedTimeIT.doSetup:63-BaseTest.setUpTestDriver:476-BaseTest.checkClusterInitialized:455-BaseTest.setUpTestCluster:469-BaseTest.initMiniCluster:519
 » Runtime
  
SpooledOrderByIT.doSetup:38-BaseTest.setUpTestDriver:476-BaseTest.checkClusterInitialized:455-BaseTest.setUpTestCluster:469-BaseTest.initMiniCluster:519
 » Runtime
  
SequenceIT.doSetup:70-BaseTest.setUpTestDriver:476-BaseTest.checkClusterInitialized:455-BaseTest.setUpTestCluster:469-BaseTest.initMiniCluster:519
 » Runtime
  
DynamicUpsertITBaseClientManagedTimeIT.doSetup:63-BaseTest.setUpTestDriver:476-BaseTest.checkClusterInitialized:455-BaseTest.setUpTestCluster:469-BaseTest.initMiniCluster:519
 » Runtime
  
ExtendedQueryExecITBaseClientManagedTimeIT.doSetup:63-BaseTest.setUpTestDriver:476-BaseTest.checkClusterInitialized:455-BaseTest.setUpTestCluster:469-BaseTest.initMiniCluster:519
 » Runtime
  
NativeHBaseTypesITBaseClientManagedTimeIT.doSetup:63-BaseTest.setUpTestDriver:476-BaseTest.checkClusterInitialized:455-BaseTest.setUpTestCluster:469-BaseTest.initMiniCluster:519
 »

[5/6] git commit: PHOENIX-1251 Salted queries with range scan become full table scans

2014-10-03 Thread jamestaylor 
PHOENIX-1251 Salted queries with range scan become full table scans


Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/5f6f80b8
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/5f6f80b8
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/5f6f80b8

Branch: refs/heads/3.0
Commit: 5f6f80b83b07609e4990eded142ff9b6f09393a5
Parents: 88c6abb
Author: James Taylor jtay...@salesforce.com
Authored: Fri Oct 3 20:55:50 2014 -0700
Committer: James Taylor jtay...@salesforce.com
Committed: Fri Oct 3 20:55:50 2014 -0700

--
 .../java/org/apache/phoenix/compile/ScanRanges.java   |  6 ++
 .../phoenix/coprocessor/MetaDataEndpointImpl.java |  6 +++---
 .../org/apache/phoenix/iterate/ParallelIterators.java |  2 +-
 .../org/apache/phoenix/schema/MetaDataClient.java | 14 +++---
 pom.xml   |  2 +-
 5 files changed, 18 insertions(+), 12 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/phoenix/blob/5f6f80b8/phoenix-core/src/main/java/org/apache/phoenix/compile/ScanRanges.java
--
diff --git 
a/phoenix-core/src/main/java/org/apache/phoenix/compile/ScanRanges.java 
b/phoenix-core/src/main/java/org/apache/phoenix/compile/ScanRanges.java
index 1bd8cef..533d752 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/compile/ScanRanges.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/ScanRanges.java
@@ -210,6 +210,9 @@ public class ScanRanges {
 public Scan intersectScan(Scan scan, final byte[] originalStartKey, final 
byte[] originalStopKey, final int keyOffset) {
 byte[] startKey = originalStartKey;
 byte[] stopKey = originalStopKey;
+if (stopKey.length  0  Bytes.compareTo(startKey, stopKey) = 0) {
+return null;
+}
 boolean mayHaveRows = false;
 // Keep the keys as they are if we have a point lookup, as we've 
already resolved the
 // salt bytes in that case.
@@ -338,6 +341,9 @@ public class ScanRanges {
 scanStopKey = prefixKey(scanStopKey, scanKeyOffset, 
prefixBytes, keyOffset);
 }
 }
+if (scanStopKey.length  0  Bytes.compareTo(scanStartKey, 
scanStopKey) = 0) {
+return null;
+}
 newScan.setStartRow(scanStartKey);
 newScan.setStopRow(scanStopKey);
 

http://git-wip-us.apache.org/repos/asf/phoenix/blob/5f6f80b8/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java
--
diff --git 
a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java
 
b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java
index f1f05be..17e5e15 100644
--- 
a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java
+++ 
b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java
@@ -482,10 +482,10 @@ public class MetaDataEndpointImpl extends 
BaseEndpointCoprocessor implements Met
 KeyValue current = result.raw()[0];
 int tableNameLength = tableNameBytes.length + 1;
 int cfOffset = current.getRowOffset() + tableNameLength;
-int cfLength = getVarCharLength(current.getRow(), cfOffset, 
current.getRowLength() - tableNameLength);
-ptr.set(current.getRow(), cfOffset, cfLength);
+int cfLength = getVarCharLength(current.getBuffer(), cfOffset, 
current.getRowLength() - tableNameLength);
+ptr.set(current.getBuffer(), cfOffset, cfLength);
 byte[] cfName = ByteUtil.copyKeyBytesIfNecessary(ptr);
-PhoenixArray array = 
(PhoenixArray)PDataType.VARBINARY_ARRAY.toObject(current.getValue(), 
current.getValueOffset(), current
+PhoenixArray array = 
(PhoenixArray)PDataType.VARBINARY_ARRAY.toObject(current.getBuffer(), 
current.getValueOffset(), current
 .getValueLength());
 if (array != null  array.getDimensions() != 0) {
 Listbyte[] guidePosts = 
Lists.newArrayListWithExpectedSize(array.getDimensions());  
  

http://git-wip-us.apache.org/repos/asf/phoenix/blob/5f6f80b8/phoenix-core/src/main/java/org/apache/phoenix/iterate/ParallelIterators.java
--
diff --git 
a/phoenix-core/src/main/java/org/apache/phoenix/iterate/ParallelIterators.java 
b/phoenix-core/src/main/java/org/apache/phoenix/iterate/ParallelIterators.java
index da8c212..81dfbb6 100644
--- 
a/phoenix-core/src/main/java/org/apache/phoenix/iterate/ParallelIterators.java
+++

[2/6] PHOENIX-1251 Salted queries with range scan become full table scans

2014-10-03 Thread jamestaylor 
http://git-wip-us.apache.org/repos/asf/phoenix/blob/88c6abb0/phoenix-core/src/main/java/org/apache/phoenix/iterate/ExplainTable.java
--
diff --git 
a/phoenix-core/src/main/java/org/apache/phoenix/iterate/ExplainTable.java 
b/phoenix-core/src/main/java/org/apache/phoenix/iterate/ExplainTable.java
index 84ae243..40a0cff 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/iterate/ExplainTable.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/iterate/ExplainTable.java
@@ -230,9 +230,11 @@ public abstract class ExplainTable {
 
 private void appendScanRow(StringBuilder buf, Bound bound) {
 ScanRanges scanRanges = context.getScanRanges();
-KeyRange minMaxRange = context.getMinMaxRange();
+// TODO: review this and potentially intersect the scan ranges
+// with the minMaxRange in ScanRanges to prevent having to do all this.
+KeyRange minMaxRange = scanRanges.getMinMaxRange();
 Iteratorbyte[] minMaxIterator = Iterators.emptyIterator();
-if (minMaxRange != null) {
+if (minMaxRange != KeyRange.EVERYTHING_RANGE) {
 RowKeySchema schema = tableRef.getTable().getRowKeySchema();
 if (!minMaxRange.isUnbound(bound)) {
 minMaxIterator = new RowKeyValueIterator(schema, 
minMaxRange.getRange(bound));
@@ -262,8 +264,7 @@ public abstract class ExplainTable {
 
 private void appendKeyRanges(StringBuilder buf) {
 ScanRanges scanRanges = context.getScanRanges();
-KeyRange minMaxRange = context.getMinMaxRange();
-if (minMaxRange == null  (scanRanges == ScanRanges.EVERYTHING || 
scanRanges == ScanRanges.NOTHING)) {
+if (scanRanges.isDegenerate() || scanRanges.isEverything()) {
 return;
 }
 buf.append( [);

http://git-wip-us.apache.org/repos/asf/phoenix/blob/88c6abb0/phoenix-core/src/main/java/org/apache/phoenix/iterate/ParallelIteratorRegionSplitterFactory.java
--
diff --git 
a/phoenix-core/src/main/java/org/apache/phoenix/iterate/ParallelIteratorRegionSplitterFactory.java
 
b/phoenix-core/src/main/java/org/apache/phoenix/iterate/ParallelIteratorRegionSplitterFactory.java
deleted file mode 100644
index 0448e46..000
--- 
a/phoenix-core/src/main/java/org/apache/phoenix/iterate/ParallelIteratorRegionSplitterFactory.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * License); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an AS IS BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.phoenix.iterate;
-
-import java.sql.SQLException;
-
-import org.apache.phoenix.compile.StatementContext;
-import org.apache.phoenix.parse.HintNode;
-import org.apache.phoenix.schema.PTable;
-
-
-/**
- * Factory class for the Region Splitter used by the project.
- */
-public class ParallelIteratorRegionSplitterFactory {
-
-public static ParallelIteratorRegionSplitter getSplitter(StatementContext 
context, PTable table, HintNode hintNode) throws SQLException {
-if (context.getScanRanges().useSkipScanFilter()) {
-return 
SkipRangeParallelIteratorRegionSplitter.getInstance(context, table, hintNode);
-}
-return DefaultParallelIteratorRegionSplitter.getInstance(context, 
table, hintNode);
-}
-}

http://git-wip-us.apache.org/repos/asf/phoenix/blob/88c6abb0/phoenix-core/src/main/java/org/apache/phoenix/iterate/ParallelIterators.java
--
diff --git 
a/phoenix-core/src/main/java/org/apache/phoenix/iterate/ParallelIterators.java 
b/phoenix-core/src/main/java/org/apache/phoenix/iterate/ParallelIterators.java
index edab575..da8c212 100644
--- 
a/phoenix-core/src/main/java/org/apache/phoenix/iterate/ParallelIterators.java
+++ 
b/phoenix-core/src/main/java/org/apache/phoenix/iterate/ParallelIterators.java
@@ -20,7 +20,6 @@ package org.apache.phoenix.iterate;
 import java.sql.SQLException;
 import java.util.ArrayList;
 import java.util.Collections;
-import java.util.Comparator;
 import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
@@ -34,26 +33,28 @@ import java.util.concurrent.ExecutorService;
 import

[4/6] git commit: PHOENIX-1251 Salted queries with range scan become full table scans

2014-10-03 Thread jamestaylor 
PHOENIX-1251 Salted queries with range scan become full table scans

Conflicts:

phoenix-core/src/it/java/org/apache/phoenix/end2end/SkipScanAfterManualSplitIT.java

phoenix-core/src/main/java/org/apache/phoenix/iterate/ParallelIterators.java

Conflicts:

phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseTenantSpecificViewIndexIT.java
phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseViewIT.java

phoenix-core/src/it/java/org/apache/phoenix/end2end/index/LocalIndexIT.java

phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexIT.java

phoenix-core/src/main/java/org/apache/phoenix/cache/ServerCacheClient.java

phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java
phoenix-core/src/main/java/org/apache/phoenix/execute/BaseQueryPlan.java
phoenix-core/src/main/java/org/apache/phoenix/filter/SkipScanFilter.java

phoenix-core/src/main/java/org/apache/phoenix/iterate/DefaultParallelIteratorRegionSplitter.java

phoenix-core/src/main/java/org/apache/phoenix/iterate/ParallelIteratorRegionSplitterFactory.java

phoenix-core/src/main/java/org/apache/phoenix/iterate/ParallelIterators.java

phoenix-core/src/main/java/org/apache/phoenix/query/StatsManagerImpl.java

phoenix-core/src/main/java/org/apache/phoenix/schema/stat/StatisticsCollector.java

phoenix-core/src/main/java/org/apache/phoenix/schema/stat/StatisticsTable.java
phoenix-core/src/main/java/org/apache/phoenix/util/ScanUtil.java


Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/88c6abb0
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/88c6abb0
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/88c6abb0

Branch: refs/heads/3.0
Commit: 88c6abb038d83a261be4a7fdc5388a20a8513a23
Parents: ff47a95
Author: James Taylor jtay...@salesforce.com
Authored: Wed Oct 1 08:49:04 2014 -0700
Committer: James Taylor jtay...@salesforce.com
Committed: Fri Oct 3 18:20:54 2014 -0700

--
 .../BaseParallelIteratorsRegionSplitterIT.java  |  90 
 .../end2end/BaseTenantSpecificViewIndexIT.java  |   7 +-
 .../org/apache/phoenix/end2end/BaseViewIT.java  |   5 +-
 ...efaultParallelIteratorsRegionSplitterIT.java | 163 ---
 .../org/apache/phoenix/end2end/DeleteIT.java|   1 +
 .../phoenix/end2end/GuidePostsLifeCycleIT.java  | 168 ---
 .../org/apache/phoenix/end2end/InListIT.java|   5 +-
 .../org/apache/phoenix/end2end/KeyOnlyIT.java   |  57 +--
 .../phoenix/end2end/MultiCfQueryExecIT.java |  73 +--
 .../phoenix/end2end/ParallelIteratorsIT.java| 172 +++
 .../org/apache/phoenix/end2end/QueryPlanIT.java | 202 
 ...ipRangeParallelIteratorRegionSplitterIT.java | 109 -
 .../end2end/SkipScanAfterManualSplitIT.java |  30 +-
 .../apache/phoenix/end2end/StatsManagerIT.java  | 198 
 .../end2end/TenantSpecificTablesDMLIT.java  |  55 +--
 .../phoenix/end2end/index/SaltedIndexIT.java|   4 +-
 .../apache/phoenix/cache/ServerCacheClient.java |  12 +-
 .../org/apache/phoenix/compile/QueryPlan.java   |   2 +
 .../org/apache/phoenix/compile/ScanRanges.java  | 370 ---
 .../phoenix/compile/StatementContext.java   |  46 +-
 .../apache/phoenix/compile/WhereOptimizer.java  |  39 +-
 .../coprocessor/MetaDataEndpointImpl.java   |  85 ++--
 .../apache/phoenix/execute/AggregatePlan.java   |   4 +-
 .../apache/phoenix/execute/BaseQueryPlan.java   | 216 +
 .../apache/phoenix/execute/BasicQueryPlan.java  | 211 -
 .../phoenix/execute/DegenerateQueryPlan.java|   2 +-
 .../apache/phoenix/execute/HashJoinPlan.java|  13 +-
 .../org/apache/phoenix/execute/ScanPlan.java|   6 +-
 .../apache/phoenix/filter/SkipScanFilter.java   |  29 +-
 .../phoenix/index/PhoenixIndexBuilder.java  |   1 -
 .../DefaultParallelIteratorRegionSplitter.java  | 174 ---
 .../apache/phoenix/iterate/ExplainTable.java|   9 +-
 .../ParallelIteratorRegionSplitterFactory.java  |  38 --
 .../phoenix/iterate/ParallelIterators.java  | 456 ++-
 ...SkipRangeParallelIteratorRegionSplitter.java |  83 
 .../apache/phoenix/jdbc/PhoenixStatement.java   |   5 +
 .../java/org/apache/phoenix/query/KeyRange.java |   8 +
 .../org/apache/phoenix/query/StatsManager.java  |  59 ---
 .../apache/phoenix/query/StatsManagerImpl.java  | 214 -
 .../schema/stat/StatisticsCollector.java|  47 +-
 .../phoenix/schema/stat/StatisticsTable.java|  18 +-
 .../java/org/apache/phoenix/util/ScanUtil.java  |  66 ++-
 .../compile/ScanRangesIntersectTest.java| 105 +
 .../apache/phoenix/compile/ScanRangesTest.java  |   2 +-
 .../phoenix/compile/WhereCompilerTest.java  |  13 +-
 .../phoenix/compile/WhereOptimizerTest.java |   5 +-
 .../query/BaseConnectionlessQueryTest.java

git commit: PHOENIX-1251 Salted queries with range scan become full table scans

2014-10-03 Thread jamestaylor 
Repository: phoenix
Updated Branches:
  refs/heads/4.0 6e89ee5dc - 2d0aeacff


PHOENIX-1251 Salted queries with range scan become full table scans


Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/2d0aeacf
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/2d0aeacf
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/2d0aeacf

Branch: refs/heads/4.0
Commit: 2d0aeacff5b38805df85f7be5731eb608abc3f4c
Parents: 6e89ee5
Author: James Taylor jtay...@salesforce.com
Authored: Fri Oct 3 21:53:19 2014 -0700
Committer: James Taylor jtay...@salesforce.com
Committed: Fri Oct 3 21:53:19 2014 -0700

--
 .../src/main/java/org/apache/phoenix/compile/ScanRanges.java   | 6 ++
 .../main/java/org/apache/phoenix/filter/SkipScanFilter.java| 3 +++
 2 files changed, 9 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/phoenix/blob/2d0aeacf/phoenix-core/src/main/java/org/apache/phoenix/compile/ScanRanges.java
--
diff --git 
a/phoenix-core/src/main/java/org/apache/phoenix/compile/ScanRanges.java 
b/phoenix-core/src/main/java/org/apache/phoenix/compile/ScanRanges.java
index 1bd8cef..4591bdb 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/compile/ScanRanges.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/ScanRanges.java
@@ -210,6 +210,9 @@ public class ScanRanges {
 public Scan intersectScan(Scan scan, final byte[] originalStartKey, final 
byte[] originalStopKey, final int keyOffset) {
 byte[] startKey = originalStartKey;
 byte[] stopKey = originalStopKey;
+if (stopKey.length  0  Bytes.compareTo(startKey, stopKey) = 0) { 
+return null; 
+}
 boolean mayHaveRows = false;
 // Keep the keys as they are if we have a point lookup, as we've 
already resolved the
 // salt bytes in that case.
@@ -338,6 +341,9 @@ public class ScanRanges {
 scanStopKey = prefixKey(scanStopKey, scanKeyOffset, 
prefixBytes, keyOffset);
 }
 }
+if (scanStopKey.length  0  Bytes.compareTo(scanStartKey, 
scanStopKey) = 0) { 
+return null; 
+}
 newScan.setStartRow(scanStartKey);
 newScan.setStopRow(scanStopKey);
 

http://git-wip-us.apache.org/repos/asf/phoenix/blob/2d0aeacf/phoenix-core/src/main/java/org/apache/phoenix/filter/SkipScanFilter.java
--
diff --git 
a/phoenix-core/src/main/java/org/apache/phoenix/filter/SkipScanFilter.java 
b/phoenix-core/src/main/java/org/apache/phoenix/filter/SkipScanFilter.java
index b964871..9e1f55a 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/filter/SkipScanFilter.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/filter/SkipScanFilter.java
@@ -149,6 +149,9 @@ public class SkipScanFilter extends FilterBase implements 
Writable {
 
 // we should either have no previous hint, or the next hint should 
always come after the previous hint
 // TODO: put this assert back after trying failing tests without it
+// Tests failing with this assert include: 
+// DeleteIT.testDeleteAllFromTableWithIndexNoAutoCommitNoSalting()
+// MutableIndexIT.testCoveredColumnUpdatesWithLocalIndex()
 //assert previousCellHint == null || 
KeyValue.COMPARATOR.compare(nextCellHint, previousCellHint)  0
 //: next hint must come after previous hint (prev= + 
previousCellHint + , next= + nextCellHint + , kv= + kv + );
 }

Jenkins build is back to normal : Phoenix | 4.0 | Hadoop1 #351

2014-10-03 Thread Apache Jenkins Server 
See https://builds.apache.org/job/Phoenix-4.0-hadoop1/351/changes

git commit: PHOENIX-1030 Change Expression.isDeterministic() to return a enum of values ALWAYS, PER_STATEMENT, PER_ROW (Thomas D'Silva)

2014-10-03 Thread jamestaylor 
Repository: phoenix
Updated Branches:
  refs/heads/master 340280b2e - b1be0f8b8


PHOENIX-1030 Change Expression.isDeterministic() to return a enum of values 
ALWAYS, PER_STATEMENT, PER_ROW (Thomas D'Silva)


Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/b1be0f8b
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/b1be0f8b
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/b1be0f8b

Branch: refs/heads/master
Commit: b1be0f8b8b5921bd6d0a91f549294eea7f27da95
Parents: 340280b
Author: James Taylor jtay...@salesforce.com
Authored: Fri Oct 3 22:30:45 2014 -0700
Committer: James Taylor jtay...@salesforce.com
Committed: Fri Oct 3 22:33:34 2014 -0700

--
 .../phoenix/compile/CreateTableCompiler.java|  4 +-
 .../phoenix/compile/ExpressionCompiler.java | 64 ++---
 .../apache/phoenix/compile/SequenceManager.java |  5 +-
 .../apache/phoenix/compile/UpsertCompiler.java  |  7 +-
 .../apache/phoenix/compile/WhereCompiler.java   |  3 +-
 .../apache/phoenix/compile/WhereOptimizer.java  |  3 +-
 .../apache/phoenix/execute/HashJoinPlan.java|  3 +-
 .../phoenix/expression/AndExpression.java   |  6 +-
 .../expression/BaseCompoundExpression.java  | 11 +--
 .../phoenix/expression/BaseExpression.java  |  6 +-
 .../expression/ComparisonExpression.java| 34 +++
 .../expression/CurrentDateTimeFunction.java |  4 +-
 .../apache/phoenix/expression/Determinism.java  | 19 
 .../apache/phoenix/expression/Expression.java   |  6 +-
 .../phoenix/expression/InListExpression.java|  4 +-
 .../phoenix/expression/IsNullExpression.java|  4 +-
 .../phoenix/expression/LiteralExpression.java   | 99 +++-
 .../phoenix/expression/NotExpression.java   |  4 +-
 .../expression/function/AggregateFunction.java  |  5 +-
 .../function/CeilDecimalExpression.java |  8 +-
 .../expression/function/CoalesceFunction.java   |  3 +-
 .../function/CountAggregateFunction.java|  4 +-
 .../function/FloorDecimalExpression.java|  8 +-
 .../function/RoundDateExpression.java   |  5 +-
 .../function/RoundDecimalExpression.java|  6 +-
 .../function/SingleAggregateFunction.java   |  4 +-
 .../apache/phoenix/parse/FunctionParseNode.java | 10 +-
 .../org/apache/phoenix/util/ExpressionUtil.java | 28 +++---
 .../phoenix/expression/DeterminismTest.java | 37 
 29 files changed, 248 insertions(+), 156 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/phoenix/blob/b1be0f8b/phoenix-core/src/main/java/org/apache/phoenix/compile/CreateTableCompiler.java
--
diff --git 
a/phoenix-core/src/main/java/org/apache/phoenix/compile/CreateTableCompiler.java
 
b/phoenix-core/src/main/java/org/apache/phoenix/compile/CreateTableCompiler.java
index 7794416..7a8ebf4 100644
--- 
a/phoenix-core/src/main/java/org/apache/phoenix/compile/CreateTableCompiler.java
+++ 
b/phoenix-core/src/main/java/org/apache/phoenix/compile/CreateTableCompiler.java
@@ -32,6 +32,7 @@ import org.apache.phoenix.exception.SQLExceptionInfo;
 import org.apache.phoenix.execute.MutationState;
 import org.apache.phoenix.expression.AndExpression;
 import org.apache.phoenix.expression.ComparisonExpression;
+import org.apache.phoenix.expression.Determinism;
 import org.apache.phoenix.expression.Expression;
 import org.apache.phoenix.expression.IsNullExpression;
 import org.apache.phoenix.expression.KeyValueColumnExpression;
@@ -257,7 +258,8 @@ public class CreateTableCompiler {
 
 @Override
 public IteratorExpression visitEnter(ComparisonExpression node) {
-if (node.getFilterOp() == CompareOp.EQUAL  
node.getChildren().get(1).isStateless()  
node.getChildren().get(1).isDeterministic()) {
+if (node.getFilterOp() == CompareOp.EQUAL  
node.getChildren().get(1).isStateless() 
+node.getChildren().get(1).getDeterminism() == 
Determinism.ALWAYS ) {
 return Iterators.singletonIterator(node.getChildren().get(0));
 }
 return super.visitEnter(node);

http://git-wip-us.apache.org/repos/asf/phoenix/blob/b1be0f8b/phoenix-core/src/main/java/org/apache/phoenix/compile/ExpressionCompiler.java
--
diff --git 
a/phoenix-core/src/main/java/org/apache/phoenix/compile/ExpressionCompiler.java 
b/phoenix-core/src/main/java/org/apache/phoenix/compile/ExpressionCompiler.java
index 1511539..573cb55 100644
--- 
a/phoenix-core/src/main/java/org/apache/phoenix/compile/ExpressionCompiler.java
+++ 
b/phoenix-core/src/main/java/org/apache/phoenix/compile/ExpressionCompiler.java
@@ -41,6 +41,7 @@ import org.apache.phoenix.expression.DecimalAddExpression;
 import

git commit: PHOENIX-1030 Change Expression.isDeterministic() to return a enum of values ALWAYS, PER_STATEMENT, PER_ROW (Thomas D'Silva)

2014-10-03 Thread jamestaylor 
Repository: phoenix
Updated Branches:
  refs/heads/3.0 80e218c24 - 9cf34400a


PHOENIX-1030 Change Expression.isDeterministic() to return a enum of values 
ALWAYS, PER_STATEMENT, PER_ROW (Thomas D'Silva)


Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/9cf34400
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/9cf34400
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/9cf34400

Branch: refs/heads/3.0
Commit: 9cf34400a1da6520e0573258287049242ed22c10
Parents: 80e218c
Author: James Taylor jtay...@salesforce.com
Authored: Fri Oct 3 22:39:36 2014 -0700
Committer: James Taylor jtay...@salesforce.com
Committed: Fri Oct 3 22:39:36 2014 -0700

--
 .../phoenix/compile/CreateTableCompiler.java|  4 +-
 .../phoenix/compile/ExpressionCompiler.java | 64 ++---
 .../apache/phoenix/compile/SequenceManager.java |  5 +-
 .../apache/phoenix/compile/UpsertCompiler.java  |  7 +-
 .../apache/phoenix/compile/WhereCompiler.java   |  3 +-
 .../apache/phoenix/compile/WhereOptimizer.java  |  3 +-
 .../apache/phoenix/execute/HashJoinPlan.java|  3 +-
 .../phoenix/expression/AndExpression.java   |  6 +-
 .../expression/BaseCompoundExpression.java  | 11 +--
 .../phoenix/expression/BaseExpression.java  |  6 +-
 .../expression/ComparisonExpression.java| 34 +++
 .../expression/CurrentDateTimeFunction.java |  4 +-
 .../apache/phoenix/expression/Determinism.java  | 19 
 .../apache/phoenix/expression/Expression.java   |  6 +-
 .../phoenix/expression/InListExpression.java|  4 +-
 .../phoenix/expression/IsNullExpression.java|  4 +-
 .../phoenix/expression/LiteralExpression.java   | 99 +++-
 .../phoenix/expression/NotExpression.java   |  4 +-
 .../expression/function/AggregateFunction.java  |  5 +-
 .../function/CeilDecimalExpression.java |  8 +-
 .../expression/function/CoalesceFunction.java   |  3 +-
 .../function/CountAggregateFunction.java|  4 +-
 .../function/FloorDecimalExpression.java|  8 +-
 .../function/RoundDateExpression.java   |  5 +-
 .../function/RoundDecimalExpression.java|  5 +-
 .../function/SingleAggregateFunction.java   |  4 +-
 .../apache/phoenix/parse/FunctionParseNode.java | 10 +-
 .../org/apache/phoenix/util/ExpressionUtil.java | 28 +++---
 .../phoenix/expression/DeterminismTest.java | 37 
 29 files changed, 248 insertions(+), 155 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/phoenix/blob/9cf34400/phoenix-core/src/main/java/org/apache/phoenix/compile/CreateTableCompiler.java
--
diff --git 
a/phoenix-core/src/main/java/org/apache/phoenix/compile/CreateTableCompiler.java
 
b/phoenix-core/src/main/java/org/apache/phoenix/compile/CreateTableCompiler.java
index 7794416..7a8ebf4 100644
--- 
a/phoenix-core/src/main/java/org/apache/phoenix/compile/CreateTableCompiler.java
+++ 
b/phoenix-core/src/main/java/org/apache/phoenix/compile/CreateTableCompiler.java
@@ -32,6 +32,7 @@ import org.apache.phoenix.exception.SQLExceptionInfo;
 import org.apache.phoenix.execute.MutationState;
 import org.apache.phoenix.expression.AndExpression;
 import org.apache.phoenix.expression.ComparisonExpression;
+import org.apache.phoenix.expression.Determinism;
 import org.apache.phoenix.expression.Expression;
 import org.apache.phoenix.expression.IsNullExpression;
 import org.apache.phoenix.expression.KeyValueColumnExpression;
@@ -257,7 +258,8 @@ public class CreateTableCompiler {
 
 @Override
 public IteratorExpression visitEnter(ComparisonExpression node) {
-if (node.getFilterOp() == CompareOp.EQUAL  
node.getChildren().get(1).isStateless()  
node.getChildren().get(1).isDeterministic()) {
+if (node.getFilterOp() == CompareOp.EQUAL  
node.getChildren().get(1).isStateless() 
+node.getChildren().get(1).getDeterminism() == 
Determinism.ALWAYS ) {
 return Iterators.singletonIterator(node.getChildren().get(0));
 }
 return super.visitEnter(node);

http://git-wip-us.apache.org/repos/asf/phoenix/blob/9cf34400/phoenix-core/src/main/java/org/apache/phoenix/compile/ExpressionCompiler.java
--
diff --git 
a/phoenix-core/src/main/java/org/apache/phoenix/compile/ExpressionCompiler.java 
b/phoenix-core/src/main/java/org/apache/phoenix/compile/ExpressionCompiler.java
index bd68ccb..41f6d83 100644
--- 
a/phoenix-core/src/main/java/org/apache/phoenix/compile/ExpressionCompiler.java
+++ 
b/phoenix-core/src/main/java/org/apache/phoenix/compile/ExpressionCompiler.java
@@ -41,6 +41,7 @@ import org.apache.phoenix.expression.DecimalAddExpression;
 import