date:20180219

[4/7] drill git commit: DRILL-6130: Fix NPE during physical plan submission for various storage plugins

2018-02-19 Thread vitalii

DRILL-6130: Fix NPE during physical plan submission for various storage plugins

1. Fixed ser / de issues for Hive, Kafka, Hbase plugins.
2. Added physical plan submission unit test for all storage plugins in contrib 
module.
3. Refactoring.

closes #1108


Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/58e4cec9
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/58e4cec9
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/58e4cec9

Branch: refs/heads/master
Commit: 58e4cec9a913e381ef0e96b072c31e34085277b3
Parents: 58f3b10
Author: Arina Ielchiieva 
Authored: Thu Feb 1 17:44:43 2018 +
Committer: Vitalii Diravka 
Committed: Fri Feb 16 20:25:25 2018 +

--
 .../drill/exec/store/hbase/HBaseGroupScan.java  |   5 +-
 .../drill/exec/store/hbase/HBaseSubScan.java|  38 +++---
 .../apache/drill/hbase/TestHBaseQueries.java|   6 +
 .../planner/sql/HivePartitionDescriptor.java|  14 +-
 ...onvertHiveParquetScanToDrillParquetScan.java |  14 +-
 .../store/hive/HiveDrillNativeParquetScan.java  |  22 ++--
 .../hive/HiveDrillNativeParquetSubScan.java |   4 +-
 .../apache/drill/exec/store/hive/HiveScan.java  |  68 ++
 .../drill/exec/store/hive/HiveSubScan.java  | 130 ++-
 .../apache/drill/exec/hive/TestHiveStorage.java |   7 +-
 .../store/jdbc/TestJdbcPluginWithMySQLIT.java   |   5 +
 .../drill/exec/store/kafka/KafkaGroupScan.java  |  40 +++---
 .../drill/exec/store/kafka/KafkaSubScan.java|  56 
 .../exec/store/kafka/KafkaQueriesTest.java  |  21 +--
 .../exec/store/kafka/MessageIteratorTest.java   |   2 +-
 .../drill/exec/store/kafka/QueryConstants.java  |  40 --
 .../drill/exec/store/kafka/TestKafkaSuit.java   |   6 +-
 .../exec/store/kafka/TestQueryConstants.java|  40 ++
 .../kafka/cluster/EmbeddedKafkaCluster.java |   4 +-
 .../drill/exec/store/kudu/KuduGroupScan.java|  39 +++---
 .../drill/exec/store/kudu/KuduSubScan.java  |  39 +++---
 .../apache/drill/store/kudu/TestKuduPlugin.java |   8 +-
 .../exec/store/mongo/TestMongoQueries.java  |   7 +
 .../store/openTSDB/TestOpenTSDBPlugin.java  |   7 +-
 .../apache/drill/exec/proto/UserBitShared.java  |  21 ++-
 .../exec/proto/beans/CoreOperatorType.java  |   4 +-
 protocol/src/main/protobuf/UserBitShared.proto  |   1 +
 27 files changed, 353 insertions(+), 295 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/drill/blob/58e4cec9/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseGroupScan.java
--
diff --git 
a/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseGroupScan.java
 
b/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseGroupScan.java
index 69c2725..1178298 100644
--- 
a/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseGroupScan.java
+++ 
b/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseGroupScan.java
@@ -160,7 +160,7 @@ public class HBaseGroupScan extends AbstractGroupScan 
implements DrillHBaseConst
   statsCalculator = new TableStatsCalculator(conn, hbaseScanSpec, 
storagePlugin.getContext().getConfig(), storagePluginConfig);
 
   boolean foundStartRegion = false;
-  regionsToScan = new TreeMap();
+  regionsToScan = new TreeMap<>();
   for (HRegionLocation regionLocation : regionLocations) {
 HRegionInfo regionInfo = regionLocation.getRegionInfo();
 if (!foundStartRegion && hbaseScanSpec.getStartRow() != null && 
hbaseScanSpec.getStartRow().length != 0 && 
!regionInfo.containsRow(hbaseScanSpec.getStartRow())) {
@@ -338,8 +338,7 @@ public class HBaseGroupScan extends AbstractGroupScan 
implements DrillHBaseConst
 assert minorFragmentId < endpointFragmentMapping.size() : String.format(
 "Mappings length [%d] should be greater than minor fragment id [%d] 
but it isn't.", endpointFragmentMapping.size(),
 minorFragmentId);
-return new HBaseSubScan(getUserName(), storagePlugin, storagePluginConfig,
-endpointFragmentMapping.get(minorFragmentId), columns);
+return new HBaseSubScan(getUserName(), storagePlugin, 
endpointFragmentMapping.get(minorFragmentId), columns);
   }
 
   @Override

http://git-wip-us.apache.org/repos/asf/drill/blob/58e4cec9/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseSubScan.java
--
diff --git 
a/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseSubScan.java
 
b/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseSubScan.java
index 0527391..bd179fb

[6/7] drill git commit: DRILL-6089: Removed ordering trait from HashJoin in planner and verified the planner does not assume HashJoin preserves ordering.

2018-02-19 Thread vitalii

DRILL-6089: Removed ordering trait from HashJoin in planner and verified the 
planner does not assume HashJoin preserves ordering.

closes #1117


Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/24a7acd4
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/24a7acd4
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/24a7acd4

Branch: refs/heads/master
Commit: 24a7acd440d29b7340a378306f339e1307892208
Parents: 20185c9
Author: Timothy Farkas 
Authored: Fri Jan 26 13:46:22 2018 -0800
Committer: Vitalii Diravka 
Committed: Fri Feb 16 20:32:54 2018 +

--
 .../exec/planner/physical/JoinPruleBase.java| 10 +++---
 .../drill/exec/planner/physical/PrelUtil.java   | 15 
 .../java/org/apache/drill/PlanTestBase.java | 37 +++-
 .../impl/join/TestHashJoinAdvanced.java | 16 +++--
 4 files changed, 60 insertions(+), 18 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/drill/blob/24a7acd4/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/JoinPruleBase.java
--
diff --git 
a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/JoinPruleBase.java
 
b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/JoinPruleBase.java
index 80e8dda..8b0d69a 100644
--- 
a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/JoinPruleBase.java
+++ 
b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/JoinPruleBase.java
@@ -116,7 +116,6 @@ public abstract class JoinPruleBase extends Prule {
 }
   }
 
-
   // Create join plan with both left and right children hash distributed. If 
the physical join type
   // is MergeJoin, a collation must be provided for both left and right child 
and the plan will contain
   // sort converter if necessary to provide the collation.
@@ -126,8 +125,6 @@ public abstract class JoinPruleBase extends Prule {
   RelCollation collationLeft, RelCollation collationRight,
   DrillDistributionTrait hashLeftPartition, DrillDistributionTrait 
hashRightPartition) throws InvalidRelException {
 
-//DrillDistributionTrait hashLeftPartition = new 
DrillDistributionTrait(DrillDistributionTrait.DistributionType.HASH_DISTRIBUTED,
 ImmutableList.copyOf(getDistributionField(join.getLeftKeys(;
-//DrillDistributionTrait hashRightPartition = new 
DrillDistributionTrait(DrillDistributionTrait.DistributionType.HASH_DISTRIBUTED,
 ImmutableList.copyOf(getDistributionField(join.getRightKeys(;
 RelTraitSet traitsLeft = null;
 RelTraitSet traitsRight = null;
 
@@ -146,7 +143,8 @@ public abstract class JoinPruleBase extends Prule {
 DrillJoinRelBase newJoin = null;
 
 if (physicalJoinType == PhysicalJoinType.HASH_JOIN) {
-  newJoin = new HashJoinPrel(join.getCluster(), traitsLeft,
+  final RelTraitSet traitSet = PrelUtil.removeCollation(traitsLeft, call);
+  newJoin = new HashJoinPrel(join.getCluster(), traitSet,
  convertedLeft, convertedRight, 
join.getCondition(),
  join.getJoinType());
 
@@ -236,7 +234,8 @@ public abstract class JoinPruleBase extends Prule {
 call.transformTo(new MergeJoinPrel(join.getCluster(), 
convertedLeft.getTraitSet(), convertedLeft,
 convertedRight, joinCondition, join.getJoinType()));
   } else if (physicalJoinType == PhysicalJoinType.HASH_JOIN) {
-call.transformTo(new HashJoinPrel(join.getCluster(), 
convertedLeft.getTraitSet(), convertedLeft,
+final RelTraitSet traitSet = 
PrelUtil.removeCollation(convertedLeft.getTraitSet(), call);
+call.transformTo(new HashJoinPrel(join.getCluster(), traitSet, 
convertedLeft,
 convertedRight, joinCondition, join.getJoinType()));
   } else if (physicalJoinType == PhysicalJoinType.NESTEDLOOP_JOIN) {
 call.transformTo(new NestedLoopJoinPrel(join.getCluster(), 
convertedLeft.getTraitSet(), convertedLeft,
@@ -245,5 +244,4 @@ public abstract class JoinPruleBase extends Prule {
 }
 
   }
-
 }

http://git-wip-us.apache.org/repos/asf/drill/blob/24a7acd4/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/PrelUtil.java
--
diff --git 
a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/PrelUtil.java
 
b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/PrelUtil.java
index 67b3066..9c0ee40 100644
--- 
a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/PrelUtil.java
+++ 
b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/PrelUtil.java
@@ -25,6 +25,7 @@ import java.util.Set;
 import

[1/7] drill git commit: DILL-6148: TestSortSpillWithException is sometimes failing.

2018-02-19 Thread vitalii

Repository: drill
Updated Branches:
  refs/heads/master 23042dec6 -> 894c0f58e


DILL-6148: TestSortSpillWithException is sometimes failing.

closes #1120


Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/5d4fbd1e
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/5d4fbd1e
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/5d4fbd1e

Branch: refs/heads/master
Commit: 5d4fbd1e65ec19778009eec81e01d29f1167
Parents: 23042de
Author: Hanumath Rao Maduri 
Authored: Fri Feb 9 14:13:04 2018 -0800
Committer: Vitalii Diravka 
Committed: Fri Feb 16 17:24:04 2018 +

--
 .../exec/physical/impl/xsort/TestSortSpillWithException.java| 5 +
 1 file changed, 5 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/drill/blob/5d4fbd1e/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/xsort/TestSortSpillWithException.java
--
diff --git 
a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/xsort/TestSortSpillWithException.java
 
b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/xsort/TestSortSpillWithException.java
index cd3f5fd..5ab29de 100644
--- 
a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/xsort/TestSortSpillWithException.java
+++ 
b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/xsort/TestSortSpillWithException.java
@@ -59,6 +59,11 @@ public class TestSortSpillWithException extends ClusterTest {
 ClusterFixtureBuilder builder = ClusterFixture.builder(dirTestWatcher)
 .configProperty(ExecConstants.EXTERNAL_SORT_SPILL_THRESHOLD, 1) // 
Unmanaged
 .configProperty(ExecConstants.EXTERNAL_SORT_SPILL_GROUP_SIZE, 1) // 
Unmanaged
+// Using EXTERNAL_SORT_MAX_MEMORY to set low values of memory for SORT 
instead
+// of lowering MAX_QUERY_MEMORY_PER_NODE_KEY because computation of 
operator memory
+// cannot go lower than MIN_MEMORY_PER_BUFFERED_OP (the default value 
of this parameter
+// is 40MB). The 40MB memory is sufficient for this testcase to run 
sort without spilling.
+.configProperty(ExecConstants.EXTERNAL_SORT_MAX_MEMORY, 10 * 1024 * 
1024)
 .sessionOption(ExecConstants.MAX_QUERY_MEMORY_PER_NODE_KEY, 60 * 1024 
* 1024) // Spill early
 // Prevent the percent-based memory rule from second-guessing the 
above.
 .sessionOption(ExecConstants.PERCENT_MEMORY_PER_QUERY_KEY, 0.0)

[7/7] drill git commit: DRILL-6164: Heap memory leak during parquet scan and OOM

2018-02-19 Thread vitalii

DRILL-6164: Heap memory leak during parquet scan and OOM

closes #1122


Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/894c0f58
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/894c0f58
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/894c0f58

Branch: refs/heads/master
Commit: 894c0f58ecff11ce53ede696f2a82308ca8333d4
Parents: 24a7acd
Author: Vlad Rozov 
Authored: Thu Feb 15 19:25:21 2018 -0800
Committer: Vitalii Diravka 
Committed: Mon Feb 19 08:57:05 2018 +

--
 .../exec/store/mapr/db/MapRDBScanBatchCreator.java |  4 ++--
 .../exec/store/hbase/HBaseScanBatchCreator.java|  4 ++--
 .../hive/HiveDrillNativeScanBatchCreator.java  |  3 ++-
 .../initilializers/DefaultReadersInitializer.java  |  4 ++--
 .../exec/store/kafka/KafkaScanBatchCreator.java|  4 ++--
 .../exec/store/kudu/KuduScanBatchCreator.java  |  4 ++--
 .../exec/store/mongo/MongoScanBatchCreator.java|  4 ++--
 .../exec/store/openTSDB/OpenTSDBBatchCreator.java  |  4 ++--
 .../apache/drill/exec/physical/impl/ScanBatch.java |  3 +++
 .../exec/store/dfs/easy/EasyFormatPlugin.java  |  3 ++-
 .../exec/store/mock/MockScanBatchCreator.java  |  4 ++--
 .../store/parquet/ParquetScanBatchCreator.java |  3 ++-
 .../exec/store/parquet2/DrillParquetReader.java| 14 +-
 .../parquet2/DrillParquetRecordMaterializer.java   | 17 +++--
 .../exec/physical/unit/MiniPlanUnitTestBase.java   |  5 +++--
 .../exec/physical/unit/PhysicalOpUnitTestBase.java |  3 ++-
 16 files changed, 50 insertions(+), 33 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/drill/blob/894c0f58/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/MapRDBScanBatchCreator.java
--
diff --git 
a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/MapRDBScanBatchCreator.java
 
b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/MapRDBScanBatchCreator.java
index d9c8ce7..c15d9ba 100644
--- 
a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/MapRDBScanBatchCreator.java
+++ 
b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/MapRDBScanBatchCreator.java
@@ -17,6 +17,7 @@
  */
 package org.apache.drill.exec.store.mapr.db;
 
+import java.util.LinkedList;
 import java.util.List;
 
 import org.apache.drill.common.exceptions.ExecutionSetupException;
@@ -31,13 +32,12 @@ import 
org.apache.drill.exec.store.mapr.db.binary.BinaryTableGroupScan;
 import org.apache.drill.exec.store.mapr.db.json.MaprDBJsonRecordReader;
 
 import com.google.common.base.Preconditions;
-import com.google.common.collect.Lists;
 
 public class MapRDBScanBatchCreator implements BatchCreator {
   @Override
   public ScanBatch getBatch(ExecutorFragmentContext context, MapRDBSubScan 
subScan, List children) throws ExecutionSetupException {
 Preconditions.checkArgument(children.isEmpty());
-List readers = Lists.newArrayList();
+List readers = new LinkedList<>();
 for(MapRDBSubScanSpec scanSpec : subScan.getRegionScanSpecList()){
   try {
 if (BinaryTableGroupScan.TABLE_BINARY.equals(subScan.getTableType())) {

http://git-wip-us.apache.org/repos/asf/drill/blob/894c0f58/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseScanBatchCreator.java
--
diff --git 
a/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseScanBatchCreator.java
 
b/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseScanBatchCreator.java
index ff9a4e4..87c64ac 100644
--- 
a/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseScanBatchCreator.java
+++ 
b/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseScanBatchCreator.java
@@ -17,6 +17,7 @@
  */
 package org.apache.drill.exec.store.hbase;
 
+import java.util.LinkedList;
 import java.util.List;
 
 import org.apache.drill.common.exceptions.ExecutionSetupException;
@@ -29,14 +30,13 @@ import org.apache.drill.exec.record.RecordBatch;
 import org.apache.drill.exec.store.RecordReader;
 
 import com.google.common.base.Preconditions;
-import com.google.common.collect.Lists;
 
 public class HBaseScanBatchCreator implements BatchCreator {
   @Override
   public ScanBatch getBatch(ExecutorFragmentContext context, HBaseSubScan 
subScan, List children)
   throws ExecutionSetupException {
 Preconditions.checkArgument(children.isEmpty());
-List readers = Lists.newArrayList();
+List readers = new LinkedList<>();
 List columns = null;
 for(HBaseSubScan.HBaseSubScanSpec scanSpec :

[2/7] drill git commit: DRILL-6143: Made FragmentsRunner's rpc timeout larger to reduce random failures and made it configurable as a SystemOption.

2018-02-19 Thread vitalii

DRILL-6143: Made FragmentsRunner's rpc timeout larger to reduce random failures 
and made it configurable as a SystemOption.

closes #1119


Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/33149059
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/33149059
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/33149059

Branch: refs/heads/master
Commit: 3314905987ee6d28c89cc5cfd8c7dbd6f36da41c
Parents: 5d4fbd1
Author: Timothy Farkas 
Authored: Thu Feb 8 15:25:59 2018 -0800
Committer: Vitalii Diravka 
Committed: Fri Feb 16 20:15:57 2018 +

--
 .../src/main/java/org/apache/drill/exec/ExecConstants.java  | 2 ++
 .../apache/drill/exec/server/options/SystemOptionManager.java   | 3 ++-
 .../org/apache/drill/exec/work/foreman/FragmentsRunner.java | 5 ++---
 exec/java-exec/src/main/resources/drill-module.conf | 1 +
 4 files changed, 7 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/drill/blob/33149059/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
--
diff --git 
a/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java 
b/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
index c949e51..f3572d8 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
@@ -59,6 +59,8 @@ public final class ExecConstants {
   public static final String CLIENT_RPC_THREADS = 
"drill.exec.rpc.user.client.threads";
   public static final String BIT_SERVER_RPC_THREADS = 
"drill.exec.rpc.bit.server.threads";
   public static final String USER_SERVER_RPC_THREADS = 
"drill.exec.rpc.user.server.threads";
+  public static final String FRAG_RUNNER_RPC_TIMEOUT = 
"drill.exec.rpc.fragrunner.timeout";
+  public static final PositiveLongValidator FRAG_RUNNER_RPC_TIMEOUT_VALIDATOR 
= new PositiveLongValidator(FRAG_RUNNER_RPC_TIMEOUT, Long.MAX_VALUE);
   public static final String TRACE_DUMP_DIRECTORY = 
"drill.exec.trace.directory";
   public static final String TRACE_DUMP_FILESYSTEM = 
"drill.exec.trace.filesystem";
   public static final String TEMP_DIRECTORIES = "drill.exec.tmp.directories";

http://git-wip-us.apache.org/repos/asf/drill/blob/33149059/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java
--
diff --git 
a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java
 
b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java
index 4dba96d..2b170e7 100644
--- 
a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java
+++ 
b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java
@@ -212,7 +212,8 @@ public class SystemOptionManager extends BaseOptionManager 
implements AutoClosea
   new OptionDefinition(ExecConstants.CPU_LOAD_AVERAGE),
   new OptionDefinition(ExecConstants.ENABLE_VECTOR_VALIDATOR),
   new OptionDefinition(ExecConstants.ENABLE_ITERATOR_VALIDATOR),
-  new OptionDefinition(ExecConstants.OUTPUT_BATCH_SIZE_VALIDATOR, new 
OptionMetaData(OptionValue.AccessibleScopes.SYSTEM, true, false))
+  new OptionDefinition(ExecConstants.OUTPUT_BATCH_SIZE_VALIDATOR, new 
OptionMetaData(OptionValue.AccessibleScopes.SYSTEM, true, false)),
+  new OptionDefinition(ExecConstants.FRAG_RUNNER_RPC_TIMEOUT_VALIDATOR, 
new OptionMetaData(OptionValue.AccessibleScopes.SYSTEM, true, true)),
 };
 
 final CaseInsensitiveMap map = 
CaseInsensitiveMap.newHashMap();

http://git-wip-us.apache.org/repos/asf/drill/blob/33149059/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/FragmentsRunner.java
--
diff --git 
a/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/FragmentsRunner.java
 
b/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/FragmentsRunner.java
index 2e5f2dd..b677576 100644
--- 
a/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/FragmentsRunner.java
+++ 
b/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/FragmentsRunner.java
@@ -24,6 +24,7 @@ import io.netty.buffer.ByteBuf;
 import org.apache.drill.common.concurrent.ExtendedLatch;
 import org.apache.drill.common.exceptions.ExecutionSetupException;
 import org.apache.drill.common.exceptions.UserException;
+import org.apache.drill.exec.ExecConstants;
 import org.apache.drill.exec.ops.FragmentContextImpl;
 import

[3/7] drill git commit: DRILL-6140: Correctly list Operators in Profiles Page

2018-02-19 Thread vitalii

DRILL-6140: Correctly list Operators in Profiles Page

Operators listed in Profiles Page don't always correspond with operator 
specified in Physical Plan.
This commit fixes that by using the PhysicalPlan as a reference, but reverts to 
the inferred names in the event of an Exchange-based operator

closes #1116


Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/58f3b104
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/58f3b104
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/58f3b104

Branch: refs/heads/master
Commit: 58f3b10464f5a3b969ce19705750cb163be9b287
Parents: 3314905
Author: Kunal Khatua 
Authored: Tue Feb 6 22:14:58 2018 -0800
Committer: Vitalii Diravka 
Committed: Fri Feb 16 20:21:06 2018 +

--
 .../server/rest/profile/OperatorWrapper.java| 17 ++--
 .../server/rest/profile/ProfileWrapper.java | 21 +++-
 2 files changed, 35 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/drill/blob/58f3b104/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/profile/OperatorWrapper.java
--
diff --git 
a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/profile/OperatorWrapper.java
 
b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/profile/OperatorWrapper.java
index 6322435..0df062f 100644
--- 
a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/profile/OperatorWrapper.java
+++ 
b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/profile/OperatorWrapper.java
@@ -45,12 +45,25 @@ public class OperatorWrapper {
   private final String operatorName;
   private final int size;
 
-  public OperatorWrapper(int major, 
List, String>> 
opsAndHostsList) {
+  public OperatorWrapper(int major, 
List, String>> 
opsAndHostsList, Map phyOperMap) {
 Preconditions.checkArgument(opsAndHostsList.size() > 0);
 this.major = major;
 firstProfile = opsAndHostsList.get(0).getLeft().getLeft();
 operatorType = CoreOperatorType.valueOf(firstProfile.getOperatorType());
-operatorName = operatorType == null ? UNKNOWN_OPERATOR : 
operatorType.toString();
+//Update Name from Physical Map
+String path = new 
OperatorPathBuilder().setMajor(major).setOperator(firstProfile).build();
+//Use Plan Extracted Operator Names if available
+String extractedOpName = phyOperMap.get(path);
+String inferredOpName = operatorType == null ? UNKNOWN_OPERATOR : 
operatorType.toString();
+//Revert to inferred names for exceptional cases
+// 1. Extracted 'FLATTEN' operator is NULL
+// 2. Extracted 'SCAN' could be a PARQUET_ROW_GROUP_SCAN, or 
KAFKA_SUB_SCAN, or etc.
+// 3. Extracted 'UNION_EXCHANGE' could be a SINGLE_SENDER or 
UNORDERED_RECEIVER
+if (extractedOpName == null || inferredOpName.contains(extractedOpName) || 
extractedOpName.endsWith("_EXCHANGE")) {
+  operatorName =  inferredOpName;
+} else {
+  operatorName =  extractedOpName;
+}
 this.opsAndHosts = opsAndHostsList;
 size = opsAndHostsList.size();
   }

http://git-wip-us.apache.org/repos/asf/drill/blob/58f3b104/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/profile/ProfileWrapper.java
--
diff --git 
a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/profile/ProfileWrapper.java
 
b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/profile/ProfileWrapper.java
index d59b464..2c964da 100644
--- 
a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/profile/ProfileWrapper.java
+++ 
b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/profile/ProfileWrapper.java
@@ -24,6 +24,7 @@ import java.util.List;
 import java.util.Map;
 
 import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.base.CaseFormat;
 import com.google.common.collect.Maps;
 import org.apache.commons.lang3.tuple.ImmutablePair;
 import org.apache.drill.common.config.DrillConfig;
@@ -57,10 +58,13 @@ public class ProfileWrapper {
   private final long majorFragmentTallyTotal;
   private final OptionList options;
   private final boolean onlyImpersonationEnabled;
+  private Map physicalOperatorMap;
 
   public ProfileWrapper(final QueryProfile profile, DrillConfig drillConfig) {
 this.profile = profile;
 this.id = QueryIdHelper.getQueryId(profile.getId());
+//Generating Operator Name map (DRILL-6140)
+generateOpMap(profile.getPlan());
 
 final List fragmentProfiles = new ArrayList<>();
 
@@

[4/7] drill git commit: DRILL-6130: Fix NPE during physical plan submission for various storage plugins

[6/7] drill git commit: DRILL-6089: Removed ordering trait from HashJoin in planner and verified the planner does not assume HashJoin preserves ordering.

[1/7] drill git commit: DILL-6148: TestSortSpillWithException is sometimes failing.

[7/7] drill git commit: DRILL-6164: Heap memory leak during parquet scan and OOM

[2/7] drill git commit: DRILL-6143: Made FragmentsRunner's rpc timeout larger to reduce random failures and made it configurable as a SystemOption.

[3/7] drill git commit: DRILL-6140: Correctly list Operators in Profiles Page

6 matches

Site Navigation

Mail list logo

Footer information