[drill] 07/08: DRILL-6381: Address review comments (part 2): fix formatting issues and add javadoc.
This is an automated email from the ASF dual-hosted git repository.
amansinha pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
commit 387bc4fefc40b645685439fd1da43f7223e5933c
Author: Aman Sinha
AuthorDate: Wed Oct 10 16:36:48 2018 -0700
DRILL-6381: Address review comments (part 2): fix formatting issues and add
javadoc.
---
.../planner/index/MapRDBFunctionalIndexInfo.java | 22 ++--
.../exec/planner/index/MapRDBIndexDescriptor.java | 4 +-
.../exec/store/mapr/db/MapRDBScanBatchCreator.java | 4 +-
.../exec/store/mapr/db/MapRDBSubScanSpec.java | 2 +-
.../store/mapr/db/json/JsonTableGroupScan.java | 17 ++-
.../db/json/JsonTableRangePartitionFunction.java | 4 +-
.../store/mapr/db/json/OjaiFunctionsProcessor.java | 2 +-
.../mapr/db/json/RestrictedJsonTableGroupScan.java | 8 +-
.../drill/exec/udf/mapr/db/DecodeFieldPath.java| 4 +-
.../drill/exec/udf/mapr/db/MatchesPlaceholder.java | 7 +-
.../exec/udf/mapr/db/NotMatchesPlaceholder.java| 7 +-
.../exec/udf/mapr/db/NotTypeOfPlaceholder.java | 7 +-
.../drill/exec/udf/mapr/db/SizeOfPlaceholder.java | 7 +-
.../drill/exec/udf/mapr/db/TypeOfPlaceholder.java | 7 +-
.../maprdb/tests/index/IndexHintPlanTest.java | 19 ++-
.../drill/maprdb/tests/index/IndexPlanTest.java| 135 +
.../drill/maprdb/tests/index/LargeTableGen.java| 34 +++---
.../maprdb/tests/index/LargeTableGenBase.java | 25 ++--
.../drill/maprdb/tests/index/StatisticsTest.java | 21 ++--
.../drill/maprdb/tests/index/TableIndexCmd.java| 26 ++--
.../drill/exec/physical/base/IndexGroupScan.java | 1 -
.../exec/physical/impl/join/RowKeyJoinBatch.java | 18 +--
.../exec/planner/index/DrillIndexDefinition.java | 22 ++--
.../exec/planner/index/DrillIndexDescriptor.java | 24 +++-
.../exec/planner/index/IndexConditionInfo.java | 44 ---
.../drill/exec/planner/index/IndexPlanUtils.java | 74 +--
.../drill/exec/planner/index/IndexSelector.java| 2 +-
.../drill/exec/planner/index/SimpleRexRemap.java | 18 +--
.../planner/index/rules/AbstractMatchFunction.java | 8 +-
.../planner/logical/DrillMergeProjectRule.java | 2 +-
30 files changed, 271 insertions(+), 304 deletions(-)
diff --git
a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/planner/index/MapRDBFunctionalIndexInfo.java
b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/planner/index/MapRDBFunctionalIndexInfo.java
index ec38636..67938f3 100644
---
a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/planner/index/MapRDBFunctionalIndexInfo.java
+++
b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/planner/index/MapRDBFunctionalIndexInfo.java
@@ -32,15 +32,15 @@ public class MapRDBFunctionalIndexInfo implements
FunctionalIndexInfo {
private boolean hasFunctionalField = false;
- //when we scan schemaPath in groupscan's columns, we check if this
column(schemaPath) should be rewritten to '$N',
- //When there are more than two functions on the same column in index,
CAST(a.b as INT), CAST(a.b as VARCHAR),
+ // When we scan schemaPath in groupscan's columns, we check if this
column(schemaPath) should be rewritten to '$N',
+ // When there are more than two functions on the same column in index,
CAST(a.b as INT), CAST(a.b as VARCHAR),
// then we should map SchemaPath a.b to a set of SchemaPath, e.g. $1, $2
private Map> columnToConvert;
// map of functional index expression to destination SchemaPath e.g. $N
private Map exprToConvert;
- //map of SchemaPath involved in a functional field
+ // map of SchemaPath involved in a functional field
private Map> pathsInExpr;
private Set newPathsForIndexedFunction;
@@ -52,7 +52,7 @@ public class MapRDBFunctionalIndexInfo implements
FunctionalIndexInfo {
columnToConvert = Maps.newHashMap();
exprToConvert = Maps.newHashMap();
pathsInExpr = Maps.newHashMap();
-//keep the order of new paths, it may be related to the naming policy
+// keep the order of new paths, it may be related to the naming policy
newPathsForIndexedFunction = Sets.newLinkedHashSet();
allPathsInFunction = Sets.newHashSet();
init();
@@ -60,15 +60,15 @@ public class MapRDBFunctionalIndexInfo implements
FunctionalIndexInfo {
private void init() {
int count = 0;
-for(LogicalExpression indexedExpr : indexDesc.getIndexColumns()) {
- if( !(indexedExpr instanceof SchemaPath) ) {
+for (LogicalExpression indexedExpr : indexDesc.getIndexColumns()) {
+ if (!(indexedExpr instanceof SchemaPath)) {
hasFunctionalField = true;
SchemaPath functionalFieldPath = SchemaPath.getSimplePath("$"+count);
newPathsForIndexedFunction.add(functionalFieldPath);
-//now we handle only cast expression
-if(indexedExpr instanceof CastExpression) {
- //We handle only CAST directly on
[drill] 05/08: DRILL-6381: (Part 5) Update Javadoc for a few interfaces.
This is an automated email from the ASF dual-hosted git repository.
amansinha pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
commit 3c2f9ab6c0ed290bed3bc440d72a670bbe512b5a
Author: Aman Sinha
AuthorDate: Thu Sep 13 03:55:47 2018 -0700
DRILL-6381: (Part 5) Update Javadoc for a few interfaces.
---
.../org/apache/drill/exec/physical/base/DbSubScan.java | 4 ++--
.../drill/exec/planner/index/IndexCollection.java | 4 +++-
.../drill/exec/planner/index/IndexDefinition.java | 18 --
.../drill/exec/planner/index/IndexDescriptor.java | 15 +++
4 files changed, 36 insertions(+), 5 deletions(-)
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/DbSubScan.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/DbSubScan.java
index 874468d..1977257 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/DbSubScan.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/DbSubScan.java
@@ -32,8 +32,8 @@ public interface DbSubScan extends SubScan {
boolean isRestrictedSubScan();
/**
- * For a restricted sub-scan, this method allows associating a (hash)join
instance. A subscan within a minor
- * fragment must have a corresponding (hash)join batch instance from which
it will retrieve its set of
+ * For a restricted sub-scan, this method allows associating a rowkey join
instance. A subscan within a minor
+ * fragment must have a corresponding rowkey join batch instance from which
it will retrieve its set of
* rowkeys to perform the restricted scan.
* @param batch
*/
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/index/IndexCollection.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/index/IndexCollection.java
index 9b4d170..f5ab173 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/index/IndexCollection.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/index/IndexCollection.java
@@ -21,7 +21,9 @@ import org.apache.calcite.rex.RexNode;
import org.apache.drill.common.expression.SchemaPath;
import org.apache.drill.exec.physical.base.IndexGroupScan;
-// Interface used to describe an index collection
+/**
+ * Interface used to describe an index collection
+ */
public interface IndexCollection extends Iterable {
/**
* Types of an index collections: NATIVE_SECONDARY_INDEX_COLLECTION,
EXTERNAL_SECONDARY_INDEX_COLLECTION
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/index/IndexDefinition.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/index/IndexDefinition.java
index 995d23c..bfbccc2 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/index/IndexDefinition.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/index/IndexDefinition.java
@@ -26,7 +26,9 @@ import java.util.Collection;
import java.util.List;
import java.util.Map;
-// Interface used to define an index,
+/**
+ * Top level interface used to define an index.
+ */
public interface IndexDefinition {
/**
* Types of an index: PRIMARY_KEY_INDEX, NATIVE_SECONDARY_INDEX,
EXTERNAL_SECONDARY_INDEX
@@ -87,13 +89,25 @@ public interface IndexDefinition {
*/
IndexType getIndexType();
-
+ /**
+ * Get the list of columns in the index key.
+ */
List getIndexColumns();
+ /**
+ * Get the list of columns that are in the 'included' or 'covered' fields.
+ */
List getNonIndexColumns();
+ /**
+ * Get the collation property (physical ordering) of the index.
+ */
RelCollation getCollation();
+ /**
+ * Get a mapping of the LogicalExpresion/SchemaPath to its field collation
+ * - e.g Ascending/Descending, Nulls First/Nulls Last
+ */
Map getCollationMap();
/**
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/index/IndexDescriptor.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/index/IndexDescriptor.java
index f355285..d43ba81 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/index/IndexDescriptor.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/index/IndexDescriptor.java
@@ -58,11 +58,26 @@ public interface IndexDescriptor extends IndexDefinition {
*/
boolean supportsFullTextSearch();
+ /**
+ * Get the functional index information associated with this index
(Functional indexes are
+ * indexes involving expressions e.g CAST(a as INT).
+ */
FunctionalIndexInfo getFunctionalInfo();
+ /**
+ * Get the total cost of index access (I/O, CPU) in the context of the
current query
+ * @param indexProps properties (metrics) of a single index in the context
of current query
+ * @param planner Planner instance
+ * @param numProjectedFields Number of projected fields
+ * @param primaryGroupScan Primary table's GroupScan
[drill] 01/08: DRILL-6381: (Part 1) Secondary Index framework
This is an automated email from the ASF dual-hosted git repository.
amansinha pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
commit 0abcbe3f36bf6c0a2b5fe07a778d201ead8dd2ce
Author: rebase
AuthorDate: Mon Feb 12 14:10:56 2018 -0800
DRILL-6381: (Part 1) Secondary Index framework
1. Secondary Index planning interfaces and abstract classes like
DBGroupScan, DbSubScan, IndexDecriptor etc.
2. Statistics and Cost model interfaces/classes: PluginCost, Statistics,
StatisticsPayload, AbstractIndexStatistics
3. ScanBatch and RecordReader to support repeatable scan
4. Secondary Index execution related interfaces: RangePartitionSender,
RowKeyJoin, PartitionFunction
5. MD-3979: Query using cast index plan fails with NPE
Co-authored-by: Aman Sinha
Co-authored-by: chunhui-shi
Co-authored-by: Gautam Parai
Co-authored-by: Padma Penumarthy
Co-authored-by: Hanumath Rao Maduri
Conflicts:
exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ScanBatch.java
exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/DrillRelOptUtil.java
exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillTable.java
protocol/src/main/java/org/apache/drill/exec/proto/UserBitShared.java
protocol/src/main/java/org/apache/drill/exec/proto/beans/CoreOperatorType.java
protocol/src/main/protobuf/UserBitShared.proto
---
.../java/org/apache/drill/exec/ExecConstants.java | 2 +
.../exec/physical/base/AbstractDbGroupScan.java| 95 +++
.../exec/physical/base/AbstractDbSubScan.java | 37 +++
.../physical/base/AbstractPhysicalVisitor.java | 12 +
.../drill/exec/physical/base/DbGroupScan.java | 129 +
.../apache/drill/exec/physical/base/DbSubScan.java | 43 +++
.../drill/exec/physical/base/IndexGroupScan.java | 76 ++
.../drill/exec/physical/base/PhysicalVisitor.java | 4 +
...{RangeSender.java => RangePartitionSender.java} | 52 ++--
.../apache/drill/exec/physical/impl/ScanBatch.java | 15 +-
.../drill/exec/physical/impl/join/RowKeyJoin.java | 79 ++
.../drill/exec/planner/common/DrillRelOptUtil.java | 17 ++
.../apache/drill/exec/planner/cost/PluginCost.java | 79 ++
.../planner/index/AbstractIndexCollection.java | 96 +++
.../planner/index/AbstractIndexDescriptor.java | 74 ++
.../planner/index/AbstractIndexStatistics.java | 51
.../drill/exec/planner/index/CollationContext.java | 37 +++
.../exec/planner/index/DrillIndexCollection.java | 75 ++
.../exec/planner/index/DrillIndexDefinition.java | 278
.../exec/planner/index/DrillIndexDescriptor.java | 110
.../exec/planner/index/FunctionalIndexInfo.java| 85 ++
.../drill/exec/planner/index/IndexCallContext.java | 76 ++
.../drill/exec/planner/index/IndexCollection.java | 99 +++
.../drill/exec/planner/index/IndexDefinition.java | 105
.../drill/exec/planner/index/IndexDescriptor.java | 68 +
.../drill/exec/planner/index/IndexDiscover.java| 23 ++
.../exec/planner/index/IndexDiscoverBase.java | 110
.../exec/planner/index/IndexDiscoverable.java | 37 +++
.../drill/exec/planner/index/IndexGroup.java | 63 +
.../drill/exec/planner/index/IndexProperties.java | 64 +
.../drill/exec/planner/index/IndexStatistics.java | 36 +++
.../exec/planner/index/IndexableExprMarker.java| 262 +++
.../index/InvalidIndexDefinitionException.java | 27 ++
.../drill/exec/planner/index/Statistics.java | 66 +
.../exec/planner/index/StatisticsPayload.java | 24 ++
.../drill/exec/planner/logical/DrillTable.java | 4 +
.../exec/planner/physical/PartitionFunction.java | 56
.../drill/exec/record/AbstractRecordBatch.java | 2 +-
.../drill/exec/store/AbstractRecordReader.java | 5 +
.../org/apache/drill/exec/store/RecordReader.java | 8 +
.../drill/exec/util/EncodedSchemaPathSet.java | 291 +
.../org/apache/drill/exec/proto/UserBitShared.java | 75 +++---
.../drill/exec/proto/beans/CoreOperatorType.java | 8 +-
protocol/src/main/protobuf/UserBitShared.proto | 3 +-
44 files changed, 2893 insertions(+), 65 deletions(-)
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
index 21e16eb..cb0fc5c 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
@@ -807,6 +807,8 @@ public final class ExecConstants {
*/
public static final String ENABLE_ITERATOR_VALIDATION =
"drill.exec.debug.validate_iterators";
+ public static final String QUERY_ROWKEYJOIN_BATCHSIZE_KEY =
"exec.query.rowkeyjoin_batchsize";
+
[drill] 08/08: DRILL-6381: Address code review comments (part 3).
This is an automated email from the ASF dual-hosted git repository.
amansinha pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
commit 7571d52eab9b961687df7d4fb845d0a297b228bb
Author: Aman Sinha
AuthorDate: Sat Oct 13 23:38:17 2018 -0700
DRILL-6381: Address code review comments (part 3).
DRILL-6381: Add missing joinControl logic for INTERSECT_DISTINCT.
- Modified HashJoin's probe phase to process INTERSECT_DISTINCT.
- NOTE: For build phase, the functionality will be same as for SemiJoin
when it is added later.
DRILL-6381: Address code review comment for intersect_distinct.
DRILL-6381: Rebase on latest master and fix compilation issues.
DRILL-6381: Generate protobuf files for C++ native client.
DRILL-6381: Use shaded Guava classes. Add more comments and Javadoc.
---
.../planner/index/MapRDBFunctionalIndexInfo.java | 4 +-
.../exec/planner/index/MapRDBIndexDiscover.java| 12 ++---
.../drill/exec/store/mapr/PluginConstants.java | 3 ++
.../exec/store/mapr/db/MapRDBFormatPlugin.java | 6 ++-
.../store/mapr/db/MapRDBPushFilterIntoScan.java| 17 ---
.../store/mapr/db/MapRDBPushLimitIntoScan.java | 2 +-
.../store/mapr/db/MapRDBPushProjectIntoScan.java | 35 ++---
.../mapr/db/MapRDBRestrictedScanBatchCreator.java | 4 +-
.../db/json/JsonTableRangePartitionFunction.java | 4 +-
.../store/mapr/db/json/MaprDBJsonRecordReader.java | 2 +-
.../mapr/db/json/RestrictedJsonTableGroupScan.java | 6 +--
contrib/native/client/src/protobuf/BitData.pb.cc | 55 ++---
contrib/native/client/src/protobuf/BitData.pb.h| 36 +-
.../native/client/src/protobuf/UserBitShared.pb.cc | 57 +++---
.../native/client/src/protobuf/UserBitShared.pb.h | 7 +--
.../exec/store/hbase/HBasePushFilterIntoScan.java | 2 +-
.../store/kafka/KafkaPushDownFilterIntoScan.java | 4 +-
.../store/mongo/MongoPushDownFilterForScan.java| 2 +-
.../apache/drill/exec/physical/impl/ScanBatch.java | 17 +++
.../exec/physical/impl/common/HashPartition.java | 10 ++--
.../physical/impl/join/HashJoinProbeTemplate.java | 24 +++--
.../exec/physical/impl/join/RowKeyJoinBatch.java | 6 +++
.../physical/impl/partitionsender/Partitioner.java | 3 ++
.../RangePartitionRecordBatch.java | 7 +++
.../exec/planner/common/DrillScanRelBase.java | 2 +-
.../index/InvalidIndexDefinitionException.java | 4 ++
.../generators/IndexIntersectPlanGenerator.java| 18 +++
.../planner/logical/DrillMergeProjectRule.java | 8 +++
.../planner/physical/BroadcastExchangePrel.java| 5 +-
.../planner/physical/ConvertCountToDirectScan.java | 5 +-
.../drill/exec/planner/physical/ScanPrel.java | 9
.../drill/exec/planner/physical/ScanPrule.java | 2 +-
.../InfoSchemaPushFilterIntoRecordGenerator.java | 2 +-
.../exec/store/parquet/ParquetPushDownFilter.java | 2 +-
.../physical/impl/common/HashPartitionTest.java| 2 +-
35 files changed, 248 insertions(+), 136 deletions(-)
diff --git
a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/planner/index/MapRDBFunctionalIndexInfo.java
b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/planner/index/MapRDBFunctionalIndexInfo.java
index 67938f3..564a037 100644
---
a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/planner/index/MapRDBFunctionalIndexInfo.java
+++
b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/planner/index/MapRDBFunctionalIndexInfo.java
@@ -17,8 +17,8 @@
*/
package org.apache.drill.exec.planner.index;
-import com.google.common.collect.Maps;
-import com.google.common.collect.Sets;
+import org.apache.drill.shaded.guava.com.google.common.collect.Maps;
+import org.apache.drill.shaded.guava.com.google.common.collect.Sets;
import org.apache.drill.common.expression.CastExpression;
import org.apache.drill.common.expression.LogicalExpression;
import org.apache.drill.common.expression.SchemaPath;
diff --git
a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/planner/index/MapRDBIndexDiscover.java
b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/planner/index/MapRDBIndexDiscover.java
index c231e11..aed3e04 100644
---
a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/planner/index/MapRDBIndexDiscover.java
+++
b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/planner/index/MapRDBIndexDiscover.java
@@ -100,7 +100,7 @@ public class MapRDBIndexDiscover extends IndexDiscoverBase
implements IndexDisco
for (IndexDesc idx : indexes) {
DrillIndexDescriptor hbaseIdx = buildIndexDescriptor(tableName, idx);
if (hbaseIdx == null) {
- //not able to build a valid index based on the index info from MFS
+ // not able to build a valid index based on the index info from MFS
logger.error("Not able to build index for
[drill] branch master updated (61e8b46 -> 7571d52)
This is an automated email from the ASF dual-hosted git repository. amansinha pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/drill.git. from 61e8b46 DRILL-6715: Update descriptions for System Options table new 0abcbe3 DRILL-6381: (Part 1) Secondary Index framework new a4f62e9 DRILL-6381: (Part 2) MapRDB plugin update to 6.0.1 new 12f57f0 DRILL-6381: (Part 3) Planner and Execution implementation to support Secondary Indexes new 2a9e51f DRILL-6381: (Part 4) Enhance MapR-DB plugin to support querying secondary indexes new 3c2f9ab DRILL-6381: (Part 5) Update Javadoc for a few interfaces. new 5fa9c80 DRILL-6381: Address code review comments. new 387bc4f DRILL-6381: Address review comments (part 2): fix formatting issues and add javadoc. new 7571d52 DRILL-6381: Address code review comments (part 3). The 8 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: contrib/format-maprdb/pom.xml | 89 ++ .../planner/index/MapRDBFunctionalIndexInfo.java | 168 ++ .../exec/planner/index/MapRDBIndexDescriptor.java | 222 +++ .../exec/planner/index/MapRDBIndexDiscover.java| 376 + .../drill/exec/planner/index/MapRDBStatistics.java | 1006 .../planner/index/MapRDBStatisticsPayload.java | 44 +- .../drill/exec/store/mapr/PluginConstants.java | 93 ++ .../drill/exec/store/mapr/PluginErrorHandler.java | 50 + .../drill/exec/store/mapr/db/MapRDBCost.java | 104 ++ .../exec/store/mapr/db/MapRDBFormatMatcher.java| 46 +- .../exec/store/mapr/db/MapRDBFormatPlugin.java | 69 +- .../store/mapr/db/MapRDBFormatPluginConfig.java|4 + .../drill/exec/store/mapr/db/MapRDBGroupScan.java | 83 +- .../store/mapr/db/MapRDBPushFilterIntoScan.java| 20 +- .../store/mapr/db/MapRDBPushLimitIntoScan.java | 203 +++ .../store/mapr/db/MapRDBPushProjectIntoScan.java | 142 ++ java => MapRDBRestrictedScanBatchCreator.java} | 41 +- .../exec/store/mapr/db/MapRDBScanBatchCreator.java |6 +- .../drill/exec/store/mapr/db/MapRDBSubScan.java| 23 +- .../exec/store/mapr/db/MapRDBSubScanSpec.java | 54 +- .../drill/exec/store/mapr/db/MapRDBTableCache.java | 232 +++ .../store/mapr/db/RestrictedMapRDBSubScan.java | 80 + .../store/mapr/db/RestrictedMapRDBSubScanSpec.java | 191 +++ .../store/mapr/db/binary/BinaryTableGroupScan.java | 34 +- .../store/mapr/db/json/AllTextValueWriter.java | 80 + .../mapr/db/json/CompareFunctionsProcessor.java| 15 +- .../mapr/db/json/DocumentReaderVectorWriter.java | 42 + .../exec/store/mapr/db/json/FieldPathHelper.java | 75 + .../mapr/db/json/FieldTransferVectorWriter.java| 49 + .../store/mapr/db/json/IdOnlyVectorWriter.java | 65 + .../store/mapr/db/json/JsonConditionBuilder.java | 59 +- .../exec/store/mapr/db/json/JsonScanSpec.java | 93 +- .../exec/store/mapr/db/json/JsonSubScanSpec.java | 69 +- .../store/mapr/db/json/JsonTableGroupScan.java | 676 +++- .../db/json/JsonTableRangePartitionFunction.java | 237 +++ .../store/mapr/db/json/MaprDBJsonRecordReader.java | 538 +++ .../mapr/db/json/NumbersAsDoubleValueWriter.java | 51 + .../store/mapr/db/json/OjaiFunctionsProcessor.java | 214 +++ .../exec/store/mapr/db/json/OjaiValueWriter.java | 194 +++ .../db/json/ProjectionPassthroughVectorWriter.java | 83 + .../mapr/db/json/RestrictedJsonRecordReader.java | 248 +++ .../mapr/db/json/RestrictedJsonTableGroupScan.java | 184 +++ .../store/mapr/db/json/RowCountVectorWriter.java | 29 +- .../exec/udf/mapr/db/ConditionPlaceholder.java | 54 + .../drill/exec/udf/mapr/db/DecodeFieldPath.java| 65 + .../drill/exec/udf/mapr/db/MatchesPlaceholder.java | 53 + .../exec/udf/mapr/db/NotMatchesPlaceholder.java| 53 + .../exec/udf/mapr/db/NotTypeOfPlaceholder.java | 53 + .../drill/exec/udf/mapr/db/SizeOfPlaceholder.java | 54 + .../drill/exec/udf/mapr/db/TypeOfPlaceholder.java | 53 + .../src/main/resources/drill-module.conf | 20 +- .../mapr/drill/maprdb/tests/MaprDBTestsSuite.java | 63 +- .../maprdb/tests/index/IndexHintPlanTest.java | 166 ++ .../drill/maprdb/tests/index/IndexPlanTest.java| 1690 .../drill/maprdb/tests/index/LargeTableGen.java| 172 ++ .../maprdb/tests/index/LargeTableGenBase.java | 185 +++ .../drill/maprdb/tests/index/StatisticsTest.java | 108 ++ .../drill/maprdb/tests/index/TableIndexCmd.java| 123 ++ .../mapr/drill/maprdb/tests/json/BaseJsonTest.java | 13 + .../maprdb/tests/json/TestEncodedFieldPaths.java | 128 ++ .../maprdb/tests/json/TestFieldPathHelper.java | 52 +
[drill] 06/08: DRILL-6381: Address code review comments.
This is an automated email from the ASF dual-hosted git repository.
amansinha pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
commit 5fa9c808daef8ea70a39ac2248daa99055955b72
Author: Aman Sinha
AuthorDate: Mon Oct 1 12:06:39 2018 -0700
DRILL-6381: Address code review comments.
---
.../planner/index/MapRDBFunctionalIndexInfo.java | 5
.../exec/planner/index/MapRDBIndexDiscover.java| 14 ++-
.../store/mapr/db/RestrictedMapRDBSubScanSpec.java | 28 --
.../db/json/JsonTableRangePartitionFunction.java | 2 +-
.../drill/exec/physical/config/HashJoinPOP.java| 12 --
.../planner/index/AbstractIndexDescriptor.java | 6 +
.../drill/exec/planner/index/IndexDescriptor.java | 11 +++--
.../drill/exec/planner/index/IndexGroup.java | 6 +
.../generators/IndexIntersectPlanGenerator.java| 22 +
.../generators/NonCoveringIndexPlanGenerator.java | 21
.../physical/DrillDistributionTraitDef.java| 8 ++-
.../exec/planner/physical/RowKeyJoinPrel.java | 7 --
12 files changed, 70 insertions(+), 72 deletions(-)
diff --git
a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/planner/index/MapRDBFunctionalIndexInfo.java
b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/planner/index/MapRDBFunctionalIndexInfo.java
index 01561a3..ec38636 100644
---
a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/planner/index/MapRDBFunctionalIndexInfo.java
+++
b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/planner/index/MapRDBFunctionalIndexInfo.java
@@ -138,6 +138,11 @@ public class MapRDBFunctionalIndexInfo implements
FunctionalIndexInfo {
}
/**
+ * Suppose the index key has functions (rather than plain columns): CAST(a
as int), CAST(b as varchar(10)),
+ * then we want to maintain a mapping of the logical expression of that
function to the schema path of the
+ * base column involved in the function. In this example map has 2 entries:
+ * CAST(a as int) --> 'a'
+ * CAST(b as varchar(10)) --> 'b'
* @return the map of indexed expression --> the involved schema paths in a
indexed expression
*/
public Map> getPathsInFunctionExpr() {
diff --git
a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/planner/index/MapRDBIndexDiscover.java
b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/planner/index/MapRDBIndexDiscover.java
index e1b8a61..c231e11 100644
---
a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/planner/index/MapRDBIndexDiscover.java
+++
b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/planner/index/MapRDBIndexDiscover.java
@@ -37,6 +37,7 @@ import org.apache.drill.common.expression.parser.ExprLexer;
import org.apache.drill.common.expression.parser.ExprParser;
import org.apache.drill.common.types.TypeProtos;
import org.apache.drill.common.types.Types;
+import org.apache.drill.common.util.DrillFileUtils;
import org.apache.drill.exec.physical.base.AbstractDbGroupScan;
import org.apache.drill.exec.physical.base.GroupScan;
import org.apache.drill.exec.planner.common.DrillScanRelBase;
@@ -67,6 +68,7 @@ import java.util.Set;
public class MapRDBIndexDiscover extends IndexDiscoverBase implements
IndexDiscover {
static final String DEFAULT_STRING_CAST_LEN_STR = "256";
+ static final String FIELD_DELIMITER = ":";
public MapRDBIndexDiscover(GroupScan inScan, DrillScanRelBase scanRel) {
super((AbstractDbGroupScan) inScan, scanRel);
@@ -78,14 +80,14 @@ public class MapRDBIndexDiscover extends IndexDiscoverBase
implements IndexDisco
@Override
public IndexCollection getTableIndex(String tableName) {
-//return getTableIndexFromCommandLine(tableName);
return getTableIndexFromMFS(tableName);
}
/**
- *
+ * For a given table name get the list of indexes defined on the table
according to the visibility of
+ * the indexes based on permissions.
* @param tableName
- * @return
+ * @return an IndexCollection representing the list of indexes for that table
*/
private IndexCollection getTableIndexFromMFS(String tableName) {
try {
@@ -120,8 +122,8 @@ public class MapRDBIndexDiscover extends IndexDiscoverBase
implements IndexDisco
FileSelection deriveFSSelection(DrillFileSystem fs, IndexDescriptor idxDesc)
throws IOException {
String tableName = idxDesc.getTableName();
-String[] tablePath = tableName.split("/");
-String tableParent = tableName.substring(0, tableName.lastIndexOf("/"));
+String[] tablePath = tableName.split(DrillFileUtils.SEPARATOR);
+String tableParent = tableName.substring(0,
tableName.lastIndexOf(DrillFileUtils.SEPARATOR));
return FileSelection.create(fs, tableParent, tablePath[tablePath.length -
1], false);
}
@@ -318,7 +320,7 @@ public class MapRDBIndexDiscover extends IndexDiscoverBase
implements IndexDisco
private
[drill] branch master updated: DRILL-6715: Update descriptions for System Options table
This is an automated email from the ASF dual-hosted git repository.
kunal pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
The following commit(s) were added to refs/heads/master by this push:
new 61e8b46 DRILL-6715: Update descriptions for System Options table
61e8b46 is described below
commit 61e8b464063299dc1f67445157a46c4939b0cace
Author: Kunal Khatua
AuthorDate: Thu Oct 25 12:08:57 2018 -0700
DRILL-6715: Update descriptions for System Options table
With introduction of DRILL-5735 , the descriptions for about half the
system options still remain missing. This commit collects descriptions review
by @bbevens
1. Update options for HashAgg/Join (@Ben-Zvi )
2. Update options for Parquet Reader/Writer (@sachouche )
3. Update options for Planners (@HanumathRao , @vdiravka , @KazydubB )
4. Update options for BatchSizing (@bitblender )
5. Update options for Planner Optimizations (@arina-ielchiieva )
6. Update options for Security & Kafka (Krystal Nguyen)
7. Update options for Misc entries (@arina-ielchiieva , @vvysotskyi )
In additional, there is a patch for
`org.apache.drill.exec.compile.ClassTransformer.scalar_replacement` , which
appears to have replaced `exec.compile.scalar_replacement`. References to the
latter have been removed to avoid confusion.
Additional changes include moving the `ClassTransformer` validator to
`ExecConstants.java`
Adding support for internal options' descriptions
Removed mention of {{Will be removed in 1.15.0}}. (Refer DRILL-6527)
---
.../java/org/apache/drill/exec/ExecConstants.java | 207 +
.../drill/exec/compile/ClassTransformer.java | 9 +-
.../exec/planner/physical/PlannerSettings.java | 31 +--
.../exec/server/options/SystemOptionManager.java | 3 +-
.../apache/drill/exec/server/rest/WebServer.java | 3 +
.../java-exec/src/main/resources/drill-module.conf | 1 -
.../java/org/apache/drill/test/QueryTestUtil.java | 6 +-
7 files changed, 158 insertions(+), 102 deletions(-)
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
index 03707b2..21e16eb 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
@@ -91,7 +91,8 @@ public final class ExecConstants {
// Based on available memory, adjust output batch size for buffered
operators by this factor.
public static final String OUTPUT_BATCH_SIZE_AVAIL_MEM_FACTOR =
"drill.exec.memory.operator.output_batch_size_avail_mem_factor";
- public static final DoubleValidator
OUTPUT_BATCH_SIZE_AVAIL_MEM_FACTOR_VALIDATOR = new
RangeDoubleValidator(OUTPUT_BATCH_SIZE_AVAIL_MEM_FACTOR, 0.01, 1.0, null);
+ public static final DoubleValidator
OUTPUT_BATCH_SIZE_AVAIL_MEM_FACTOR_VALIDATOR = new
RangeDoubleValidator(OUTPUT_BATCH_SIZE_AVAIL_MEM_FACTOR, 0.01, 1.0,
+ new OptionDescription("Based on the available system memory, adjusts the
output batch size for buffered operators by the factor set."));
// External Sort Boot configuration
@@ -115,25 +116,35 @@ public final class ExecConstants {
// Hash Join Options
public static final String HASHJOIN_HASHTABLE_CALC_TYPE_KEY =
"exec.hashjoin.hash_table_calc_type";
- public static final StringValidator HASHJOIN_HASHTABLE_CALC_TYPE = new
StringValidator(HASHJOIN_HASHTABLE_CALC_TYPE_KEY, null);
+ public static final EnumeratedStringValidator HASHJOIN_HASHTABLE_CALC_TYPE =
new EnumeratedStringValidator(HASHJOIN_HASHTABLE_CALC_TYPE_KEY,
+ new OptionDescription("Sets the Hash Join Memory Calculator type.
Default is LEAN. This option also accepts CONSERVATIVE as a value."),
+ "LEAN", "CONSERVATIVE");
public static final String HASHJOIN_SAFETY_FACTOR_KEY =
"exec.hashjoin.safety_factor";
- public static final DoubleValidator HASHJOIN_SAFETY_FACTOR = new
RangeDoubleValidator(HASHJOIN_SAFETY_FACTOR_KEY, 1.0, Double.MAX_VALUE, null);
+ public static final DoubleValidator HASHJOIN_SAFETY_FACTOR = new
RangeDoubleValidator(HASHJOIN_SAFETY_FACTOR_KEY, 1.0, Double.MAX_VALUE,
+ new OptionDescription("Sets the Hash Join Memory Calculation Safety;
multiplies the internal size estimate. Default is 1.0"));
public static final String HASHJOIN_HASH_DOUBLE_FACTOR_KEY =
"exec.hashjoin.hash_double_factor";
- public static final DoubleValidator HASHJOIN_HASH_DOUBLE_FACTOR = new
RangeDoubleValidator(HASHJOIN_HASH_DOUBLE_FACTOR_KEY, 1.0, Double.MAX_VALUE,
null);
+ public static final DoubleValidator HASHJOIN_HASH_DOUBLE_FACTOR = new
RangeDoubleValidator(HASHJOIN_HASH_DOUBLE_FACTOR_KEY, 1.0, Double.MAX_VALUE,
+ new OptionDescription("Sets the Hash Join Memory Calculation; doubling
factor for the Hash-Table. Default is 2.0"));
public static final String
