HIVE-9152 - Dynamic Partition Pruning [Spark Branch] (Chao Sun, reviewed by Xuefu Zhang and Chengxiang Li)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/42216997 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/42216997 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/42216997 Branch: refs/heads/spark Commit: 42216997f7fcff1853524b03e3961ec5c21f3fd7 Parents: dddffe1 Author: Chao Sun <[email protected]> Authored: Fri Jul 17 12:51:27 2015 -0700 Committer: Chao Sun <[email protected]> Committed: Fri Jul 17 12:51:27 2015 -0700 ---------------------------------------------------------------------- .../org/apache/hadoop/hive/conf/HiveConf.java | 7 + itests/qtest/pom.xml | 2 +- .../test/resources/testconfiguration.properties | 6 + .../gen/thrift/gen-cpp/ThriftHiveMetastore.cpp | 4708 ++++- .../gen/thrift/gen-cpp/ThriftHiveMetastore.h | 7807 +++---- .../thrift/gen-cpp/hive_metastore_constants.cpp | 2 +- .../thrift/gen-cpp/hive_metastore_constants.h | 2 +- .../gen/thrift/gen-cpp/hive_metastore_types.cpp | 6044 +++++- .../gen/thrift/gen-cpp/hive_metastore_types.h | 2619 ++- .../hive/metastore/api/AbortTxnRequest.java | 24 +- .../metastore/api/AddDynamicPartitions.java | 69 +- .../metastore/api/AddPartitionsRequest.java | 80 +- .../hive/metastore/api/AddPartitionsResult.java | 48 +- .../hadoop/hive/metastore/api/AggrStats.java | 54 +- .../metastore/api/AlreadyExistsException.java | 24 +- .../metastore/api/BinaryColumnStatsData.java | 40 +- .../metastore/api/BooleanColumnStatsData.java | 40 +- .../hive/metastore/api/CheckLockRequest.java | 24 +- .../hive/metastore/api/ColumnStatistics.java | 54 +- .../metastore/api/ColumnStatisticsData.java | 20 +- .../metastore/api/ColumnStatisticsDesc.java | 58 +- .../hive/metastore/api/ColumnStatisticsObj.java | 40 +- .../hive/metastore/api/CommitTxnRequest.java | 24 +- .../hive/metastore/api/CompactionRequest.java | 62 +- .../hive/metastore/api/CompactionType.java | 2 +- .../api/ConfigValSecurityException.java | 24 +- .../api/CurrentNotificationEventId.java | 24 +- .../hadoop/hive/metastore/api/Database.java | 115 +- .../apache/hadoop/hive/metastore/api/Date.java | 24 +- .../hive/metastore/api/DateColumnStatsData.java | 50 +- .../hadoop/hive/metastore/api/Decimal.java | 41 +- .../metastore/api/DecimalColumnStatsData.java | 50 +- .../metastore/api/DoubleColumnStatsData.java | 50 +- .../hive/metastore/api/DropPartitionsExpr.java | 43 +- .../metastore/api/DropPartitionsRequest.java | 82 +- .../metastore/api/DropPartitionsResult.java | 48 +- .../hive/metastore/api/EnvironmentContext.java | 61 +- .../hive/metastore/api/EventRequestType.java | 2 +- .../hadoop/hive/metastore/api/FieldSchema.java | 58 +- .../hive/metastore/api/FireEventRequest.java | 79 +- .../metastore/api/FireEventRequestData.java | 20 +- .../hive/metastore/api/FireEventResponse.java | 16 +- .../hadoop/hive/metastore/api/Function.java | 110 +- .../hadoop/hive/metastore/api/FunctionType.java | 2 +- .../metastore/api/GetOpenTxnsInfoResponse.java | 54 +- .../hive/metastore/api/GetOpenTxnsResponse.java | 53 +- .../api/GetPrincipalsInRoleRequest.java | 24 +- .../api/GetPrincipalsInRoleResponse.java | 46 +- .../api/GetRoleGrantsForPrincipalRequest.java | 36 +- .../api/GetRoleGrantsForPrincipalResponse.java | 46 +- .../api/GrantRevokePrivilegeRequest.java | 46 +- .../api/GrantRevokePrivilegeResponse.java | 26 +- .../metastore/api/GrantRevokeRoleRequest.java | 86 +- .../metastore/api/GrantRevokeRoleResponse.java | 26 +- .../hive/metastore/api/GrantRevokeType.java | 2 +- .../hive/metastore/api/HeartbeatRequest.java | 34 +- .../metastore/api/HeartbeatTxnRangeRequest.java | 32 +- .../api/HeartbeatTxnRangeResponse.java | 74 +- .../hive/metastore/api/HiveObjectPrivilege.java | 52 +- .../hive/metastore/api/HiveObjectRef.java | 81 +- .../hive/metastore/api/HiveObjectType.java | 2 +- .../apache/hadoop/hive/metastore/api/Index.java | 133 +- .../api/IndexAlreadyExistsException.java | 24 +- .../metastore/api/InsertEventRequestData.java | 45 +- .../metastore/api/InvalidInputException.java | 24 +- .../metastore/api/InvalidObjectException.java | 24 +- .../api/InvalidOperationException.java | 24 +- .../api/InvalidPartitionException.java | 24 +- .../hive/metastore/api/LockComponent.java | 66 +- .../hadoop/hive/metastore/api/LockLevel.java | 2 +- .../hadoop/hive/metastore/api/LockRequest.java | 72 +- .../hadoop/hive/metastore/api/LockResponse.java | 36 +- .../hadoop/hive/metastore/api/LockState.java | 2 +- .../hadoop/hive/metastore/api/LockType.java | 2 +- .../hive/metastore/api/LongColumnStatsData.java | 50 +- .../hive/metastore/api/MetaException.java | 24 +- .../hive/metastore/api/NoSuchLockException.java | 24 +- .../metastore/api/NoSuchObjectException.java | 24 +- .../hive/metastore/api/NoSuchTxnException.java | 24 +- .../hive/metastore/api/NotificationEvent.java | 66 +- .../metastore/api/NotificationEventRequest.java | 34 +- .../api/NotificationEventResponse.java | 46 +- .../hive/metastore/api/OpenTxnRequest.java | 40 +- .../hive/metastore/api/OpenTxnsResponse.java | 45 +- .../apache/hadoop/hive/metastore/api/Order.java | 32 +- .../hadoop/hive/metastore/api/Partition.java | 156 +- .../hive/metastore/api/PartitionEventType.java | 2 +- .../api/PartitionListComposingSpec.java | 46 +- .../hive/metastore/api/PartitionSpec.java | 58 +- .../api/PartitionSpecWithSharedSD.java | 54 +- .../hive/metastore/api/PartitionWithoutSD.java | 124 +- .../metastore/api/PartitionsByExprRequest.java | 67 +- .../metastore/api/PartitionsByExprResult.java | 54 +- .../metastore/api/PartitionsStatsRequest.java | 90 +- .../metastore/api/PartitionsStatsResult.java | 72 +- .../metastore/api/PrincipalPrivilegeSet.java | 184 +- .../hive/metastore/api/PrincipalType.java | 2 +- .../hadoop/hive/metastore/api/PrivilegeBag.java | 46 +- .../hive/metastore/api/PrivilegeGrantInfo.java | 60 +- .../hive/metastore/api/RequestPartsSpec.java | 56 +- .../hadoop/hive/metastore/api/ResourceType.java | 2 +- .../hadoop/hive/metastore/api/ResourceUri.java | 36 +- .../apache/hadoop/hive/metastore/api/Role.java | 40 +- .../hive/metastore/api/RolePrincipalGrant.java | 80 +- .../hadoop/hive/metastore/api/Schema.java | 91 +- .../hadoop/hive/metastore/api/SerDeInfo.java | 93 +- .../api/SetPartitionsStatsRequest.java | 46 +- .../hive/metastore/api/ShowCompactRequest.java | 16 +- .../hive/metastore/api/ShowCompactResponse.java | 46 +- .../api/ShowCompactResponseElement.java | 86 +- .../hive/metastore/api/ShowLocksRequest.java | 16 +- .../hive/metastore/api/ShowLocksResponse.java | 46 +- .../metastore/api/ShowLocksResponseElement.java | 114 +- .../hadoop/hive/metastore/api/SkewedInfo.java | 147 +- .../hive/metastore/api/StorageDescriptor.java | 242 +- .../metastore/api/StringColumnStatsData.java | 48 +- .../apache/hadoop/hive/metastore/api/Table.java | 189 +- .../hive/metastore/api/TableStatsRequest.java | 61 +- .../hive/metastore/api/TableStatsResult.java | 46 +- .../hive/metastore/api/ThriftHiveMetastore.java | 18497 ++++++++++++----- .../hive/metastore/api/TxnAbortedException.java | 24 +- .../hadoop/hive/metastore/api/TxnInfo.java | 52 +- .../hive/metastore/api/TxnOpenException.java | 24 +- .../hadoop/hive/metastore/api/TxnState.java | 2 +- .../apache/hadoop/hive/metastore/api/Type.java | 72 +- .../hive/metastore/api/UnknownDBException.java | 24 +- .../api/UnknownPartitionException.java | 24 +- .../metastore/api/UnknownTableException.java | 24 +- .../hive/metastore/api/UnlockRequest.java | 24 +- .../hadoop/hive/metastore/api/Version.java | 32 +- .../metastore/api/hive_metastoreConstants.java | 7 +- .../gen-php/metastore/ThriftHiveMetastore.php | 3328 ++- .../src/gen/thrift/gen-php/metastore/Types.php | 1081 +- .../hive_metastore/ThriftHiveMetastore-remote | 609 +- .../hive_metastore/ThriftHiveMetastore.py | 3726 +++- .../thrift/gen-py/hive_metastore/constants.py | 2 +- .../gen/thrift/gen-py/hive_metastore/ttypes.py | 782 +- .../thrift/gen-rb/hive_metastore_constants.rb | 2 +- .../gen/thrift/gen-rb/hive_metastore_types.rb | 2 +- .../gen/thrift/gen-rb/thrift_hive_metastore.rb | 2 +- ql/if/queryplan.thrift | 1 + .../gen/thrift/gen-cpp/queryplan_constants.cpp | 2 +- ql/src/gen/thrift/gen-cpp/queryplan_constants.h | 2 +- ql/src/gen/thrift/gen-cpp/queryplan_types.cpp | 796 +- ql/src/gen/thrift/gen-cpp/queryplan_types.h | 294 +- .../hadoop/hive/ql/plan/api/Adjacency.java | 65 +- .../hadoop/hive/ql/plan/api/AdjacencyType.java | 2 +- .../apache/hadoop/hive/ql/plan/api/Graph.java | 87 +- .../hadoop/hive/ql/plan/api/NodeType.java | 2 +- .../hadoop/hive/ql/plan/api/Operator.java | 142 +- .../hadoop/hive/ql/plan/api/OperatorType.java | 7 +- .../apache/hadoop/hive/ql/plan/api/Query.java | 176 +- .../hadoop/hive/ql/plan/api/QueryPlan.java | 62 +- .../apache/hadoop/hive/ql/plan/api/Stage.java | 172 +- .../hadoop/hive/ql/plan/api/StageType.java | 2 +- .../apache/hadoop/hive/ql/plan/api/Task.java | 182 +- .../hadoop/hive/ql/plan/api/TaskType.java | 2 +- ql/src/gen/thrift/gen-php/Types.php | 119 +- ql/src/gen/thrift/gen-py/queryplan/constants.py | 2 +- ql/src/gen/thrift/gen-py/queryplan/ttypes.py | 87 +- ql/src/gen/thrift/gen-rb/queryplan_constants.rb | 2 +- ql/src/gen/thrift/gen-rb/queryplan_types.rb | 7 +- .../hadoop/hive/ql/exec/OperatorFactory.java | 8 + .../ql/exec/spark/HiveSparkClientFactory.java | 5 +- .../exec/spark/SparkDynamicPartitionPruner.java | 268 + .../hive/ql/exec/spark/SparkUtilities.java | 56 + ...VectorSparkPartitionPruningSinkOperator.java | 99 + .../hive/ql/io/CombineHiveInputFormat.java | 59 +- .../hadoop/hive/ql/io/HiveInputFormat.java | 46 +- .../DynamicPartitionPruningOptimization.java | 44 +- .../hadoop/hive/ql/optimizer/Optimizer.java | 2 +- .../SparkRemoveDynamicPruningBySize.java | 73 + .../hive/ql/optimizer/physical/Vectorizer.java | 2 + .../spark/SparkPartitionPruningSinkDesc.java | 100 + .../ql/parse/spark/GenSparkProcContext.java | 14 +- .../hive/ql/parse/spark/GenSparkUtils.java | 111 +- .../parse/spark/OptimizeSparkProcContext.java | 16 +- .../hive/ql/parse/spark/SparkCompiler.java | 177 +- .../SparkPartitionPruningSinkOperator.java | 142 + .../hive/ql/parse/spark/SplitOpTreeForDPP.java | 151 + .../org/apache/hadoop/hive/ql/plan/MapWork.java | 10 + .../hadoop/hive/ql/plan/TableScanDesc.java | 6 +- .../hive/ql/ppd/SyntheticJoinPredicate.java | 14 +- .../spark_dynamic_partition_pruning.q | 180 + .../spark_dynamic_partition_pruning_2.q | 118 + ...spark_vectorized_dynamic_partition_pruning.q | 192 + .../results/clientpositive/spark/bucket2.q.out | 3 - .../results/clientpositive/spark/bucket3.q.out | 3 - .../results/clientpositive/spark/bucket4.q.out | 3 - .../spark/column_access_stats.q.out | 4 - .../spark/limit_partition_metadataonly.q.out | 2 - .../spark/list_bucket_dml_2.q.java1.7.out | 3 - .../spark/optimize_nullscan.q.out | 3 - .../test/results/clientpositive/spark/pcr.q.out | 6 - .../results/clientpositive/spark/sample3.q.out | 3 - .../results/clientpositive/spark/sample9.q.out | 3 - .../clientpositive/spark/smb_mapjoin_11.q.out | 6 - .../spark/spark_dynamic_partition_pruning.q.out | 5573 +++++ .../spark_dynamic_partition_pruning_2.q.out | 1015 + ...k_vectorized_dynamic_partition_pruning.q.out | 5822 ++++++ .../clientpositive/spark/udf_example_add.q.out | 3 - .../clientpositive/spark/udf_in_file.q.out | 3 - .../clientpositive/spark/union_view.q.out | 9 - .../clientpositive/spark/vector_elt.q.out | 7 - .../spark/vector_string_concat.q.out | 3 - .../spark/vectorization_decimal_date.q.out | 4 - .../spark/vectorization_div0.q.out | 3 - .../clientpositive/spark/vectorized_case.q.out | 3 - .../spark/vectorized_math_funcs.q.out | 3 - .../spark/vectorized_string_funcs.q.out | 3 - .../gen/thrift/gen-cpp/complex_constants.cpp | 2 +- .../src/gen/thrift/gen-cpp/complex_constants.h | 2 +- serde/src/gen/thrift/gen-cpp/complex_types.cpp | 442 +- serde/src/gen/thrift/gen-cpp/complex_types.h | 174 +- .../gen/thrift/gen-cpp/megastruct_constants.cpp | 2 +- .../gen/thrift/gen-cpp/megastruct_constants.h | 2 +- .../src/gen/thrift/gen-cpp/megastruct_types.cpp | 585 +- serde/src/gen/thrift/gen-cpp/megastruct_types.h | 175 +- .../src/gen/thrift/gen-cpp/serde_constants.cpp | 2 +- serde/src/gen/thrift/gen-cpp/serde_constants.h | 2 +- serde/src/gen/thrift/gen-cpp/serde_types.cpp | 5 +- serde/src/gen/thrift/gen-cpp/serde_types.h | 5 +- .../gen/thrift/gen-cpp/testthrift_constants.cpp | 2 +- .../gen/thrift/gen-cpp/testthrift_constants.h | 2 +- .../src/gen/thrift/gen-cpp/testthrift_types.cpp | 95 +- serde/src/gen/thrift/gen-cpp/testthrift_types.h | 45 +- .../hadoop/hive/serde/serdeConstants.java | 7 +- .../hadoop/hive/serde/test/InnerStruct.java | 24 +- .../hadoop/hive/serde/test/ThriftTestObj.java | 62 +- .../hadoop/hive/serde2/thrift/test/Complex.java | 279 +- .../hive/serde2/thrift/test/IntString.java | 40 +- .../hive/serde2/thrift/test/MegaStruct.java | 521 +- .../hive/serde2/thrift/test/MiniStruct.java | 38 +- .../hadoop/hive/serde2/thrift/test/MyEnum.java | 2 +- .../hive/serde2/thrift/test/PropValueUnion.java | 60 +- .../hive/serde2/thrift/test/SetIntString.java | 54 +- serde/src/gen/thrift/gen-php/Types.php | 15 +- .../org/apache/hadoop/hive/serde/Types.php | 373 +- .../src/gen/thrift/gen-py/complex/constants.py | 2 +- serde/src/gen/thrift/gen-py/complex/ttypes.py | 50 +- .../gen/thrift/gen-py/megastruct/constants.py | 2 +- .../src/gen/thrift/gen-py/megastruct/ttypes.py | 44 +- .../org_apache_hadoop_hive_serde/constants.py | 2 +- .../org_apache_hadoop_hive_serde/ttypes.py | 2 +- .../gen/thrift/gen-py/testthrift/constants.py | 2 +- .../src/gen/thrift/gen-py/testthrift/ttypes.py | 14 +- .../src/gen/thrift/gen-rb/complex_constants.rb | 2 +- serde/src/gen/thrift/gen-rb/complex_types.rb | 2 +- .../gen/thrift/gen-rb/megastruct_constants.rb | 2 +- serde/src/gen/thrift/gen-rb/megastruct_types.rb | 2 +- serde/src/gen/thrift/gen-rb/serde_constants.rb | 2 +- serde/src/gen/thrift/gen-rb/serde_types.rb | 2 +- .../gen/thrift/gen-rb/testthrift_constants.rb | 2 +- serde/src/gen/thrift/gen-rb/testthrift_types.rb | 2 +- service/src/gen/thrift/gen-cpp/TCLIService.cpp | 458 +- service/src/gen/thrift/gen-cpp/TCLIService.h | 821 +- .../thrift/gen-cpp/TCLIService_constants.cpp | 2 +- .../gen/thrift/gen-cpp/TCLIService_constants.h | 2 +- .../gen/thrift/gen-cpp/TCLIService_types.cpp | 3250 ++- .../src/gen/thrift/gen-cpp/TCLIService_types.h | 1482 +- service/src/gen/thrift/gen-cpp/ThriftHive.cpp | 286 +- service/src/gen/thrift/gen-cpp/ThriftHive.h | 389 +- .../thrift/gen-cpp/hive_service_constants.cpp | 2 +- .../gen/thrift/gen-cpp/hive_service_constants.h | 2 +- .../gen/thrift/gen-cpp/hive_service_types.cpp | 110 +- .../src/gen/thrift/gen-cpp/hive_service_types.h | 75 +- .../hadoop/hive/service/HiveClusterStatus.java | 68 +- .../hive/service/HiveServerException.java | 40 +- .../hadoop/hive/service/JobTrackerState.java | 2 +- .../apache/hadoop/hive/service/ThriftHive.java | 914 +- .../service/cli/thrift/TArrayTypeEntry.java | 24 +- .../hive/service/cli/thrift/TBinaryColumn.java | 64 +- .../hive/service/cli/thrift/TBoolColumn.java | 62 +- .../hive/service/cli/thrift/TBoolValue.java | 26 +- .../hive/service/cli/thrift/TByteColumn.java | 62 +- .../hive/service/cli/thrift/TByteValue.java | 26 +- .../hive/service/cli/thrift/TCLIService.java | 1734 +- .../cli/thrift/TCLIServiceConstants.java | 7 +- .../cli/thrift/TCancelDelegationTokenReq.java | 32 +- .../cli/thrift/TCancelDelegationTokenResp.java | 24 +- .../service/cli/thrift/TCancelOperationReq.java | 24 +- .../cli/thrift/TCancelOperationResp.java | 24 +- .../service/cli/thrift/TCloseOperationReq.java | 24 +- .../service/cli/thrift/TCloseOperationResp.java | 24 +- .../service/cli/thrift/TCloseSessionReq.java | 24 +- .../service/cli/thrift/TCloseSessionResp.java | 24 +- .../apache/hive/service/cli/thrift/TColumn.java | 20 +- .../hive/service/cli/thrift/TColumnDesc.java | 50 +- .../hive/service/cli/thrift/TColumnValue.java | 20 +- .../hive/service/cli/thrift/TDoubleColumn.java | 62 +- .../hive/service/cli/thrift/TDoubleValue.java | 26 +- .../cli/thrift/TExecuteStatementReq.java | 87 +- .../cli/thrift/TExecuteStatementResp.java | 34 +- .../service/cli/thrift/TFetchOrientation.java | 2 +- .../service/cli/thrift/TFetchResultsReq.java | 54 +- .../service/cli/thrift/TFetchResultsResp.java | 42 +- .../service/cli/thrift/TGetCatalogsReq.java | 24 +- .../service/cli/thrift/TGetCatalogsResp.java | 34 +- .../hive/service/cli/thrift/TGetColumnsReq.java | 58 +- .../service/cli/thrift/TGetColumnsResp.java | 34 +- .../cli/thrift/TGetDelegationTokenReq.java | 40 +- .../cli/thrift/TGetDelegationTokenResp.java | 34 +- .../service/cli/thrift/TGetFunctionsReq.java | 50 +- .../service/cli/thrift/TGetFunctionsResp.java | 34 +- .../hive/service/cli/thrift/TGetInfoReq.java | 36 +- .../hive/service/cli/thrift/TGetInfoResp.java | 32 +- .../hive/service/cli/thrift/TGetInfoType.java | 2 +- .../hive/service/cli/thrift/TGetInfoValue.java | 20 +- .../cli/thrift/TGetOperationStatusReq.java | 24 +- .../cli/thrift/TGetOperationStatusResp.java | 62 +- .../cli/thrift/TGetResultSetMetadataReq.java | 24 +- .../cli/thrift/TGetResultSetMetadataResp.java | 34 +- .../hive/service/cli/thrift/TGetSchemasReq.java | 42 +- .../service/cli/thrift/TGetSchemasResp.java | 34 +- .../service/cli/thrift/TGetTableTypesReq.java | 24 +- .../service/cli/thrift/TGetTableTypesResp.java | 34 +- .../hive/service/cli/thrift/TGetTablesReq.java | 79 +- .../hive/service/cli/thrift/TGetTablesResp.java | 34 +- .../service/cli/thrift/TGetTypeInfoReq.java | 24 +- .../service/cli/thrift/TGetTypeInfoResp.java | 34 +- .../service/cli/thrift/THandleIdentifier.java | 50 +- .../hive/service/cli/thrift/TI16Column.java | 62 +- .../hive/service/cli/thrift/TI16Value.java | 26 +- .../hive/service/cli/thrift/TI32Column.java | 62 +- .../hive/service/cli/thrift/TI32Value.java | 26 +- .../hive/service/cli/thrift/TI64Column.java | 62 +- .../hive/service/cli/thrift/TI64Value.java | 26 +- .../hive/service/cli/thrift/TMapTypeEntry.java | 32 +- .../service/cli/thrift/TOpenSessionReq.java | 91 +- .../service/cli/thrift/TOpenSessionResp.java | 91 +- .../service/cli/thrift/TOperationHandle.java | 54 +- .../service/cli/thrift/TOperationState.java | 2 +- .../hive/service/cli/thrift/TOperationType.java | 2 +- .../service/cli/thrift/TPrimitiveTypeEntry.java | 38 +- .../service/cli/thrift/TProtocolVersion.java | 2 +- .../cli/thrift/TRenewDelegationTokenReq.java | 32 +- .../cli/thrift/TRenewDelegationTokenResp.java | 24 +- .../apache/hive/service/cli/thrift/TRow.java | 46 +- .../apache/hive/service/cli/thrift/TRowSet.java | 86 +- .../hive/service/cli/thrift/TSessionHandle.java | 24 +- .../apache/hive/service/cli/thrift/TStatus.java | 83 +- .../hive/service/cli/thrift/TStatusCode.java | 2 +- .../hive/service/cli/thrift/TStringColumn.java | 62 +- .../hive/service/cli/thrift/TStringValue.java | 26 +- .../service/cli/thrift/TStructTypeEntry.java | 50 +- .../hive/service/cli/thrift/TTableSchema.java | 46 +- .../hive/service/cli/thrift/TTypeDesc.java | 46 +- .../hive/service/cli/thrift/TTypeEntry.java | 20 +- .../apache/hive/service/cli/thrift/TTypeId.java | 2 +- .../service/cli/thrift/TTypeQualifierValue.java | 20 +- .../service/cli/thrift/TTypeQualifiers.java | 54 +- .../service/cli/thrift/TUnionTypeEntry.java | 50 +- .../cli/thrift/TUserDefinedTypeEntry.java | 24 +- service/src/gen/thrift/gen-php/TCLIService.php | 269 +- service/src/gen/thrift/gen-php/ThriftHive.php | 125 +- service/src/gen/thrift/gen-php/Types.php | 30 +- .../gen-py/TCLIService/TCLIService-remote | 102 +- .../thrift/gen-py/TCLIService/TCLIService.py | 439 +- .../gen/thrift/gen-py/TCLIService/constants.py | 2 +- .../src/gen/thrift/gen-py/TCLIService/ttypes.py | 478 +- .../gen-py/hive_service/ThriftHive-remote | 1014 +- .../thrift/gen-py/hive_service/ThriftHive.py | 266 +- .../gen/thrift/gen-py/hive_service/constants.py | 2 +- .../gen/thrift/gen-py/hive_service/ttypes.py | 19 +- .../gen/thrift/gen-rb/hive_service_constants.rb | 2 +- .../src/gen/thrift/gen-rb/hive_service_types.rb | 2 +- .../src/gen/thrift/gen-rb/t_c_l_i_service.rb | 2 +- .../thrift/gen-rb/t_c_l_i_service_constants.rb | 2 +- .../gen/thrift/gen-rb/t_c_l_i_service_types.rb | 2 +- service/src/gen/thrift/gen-rb/thrift_hive.rb | 2 +- 370 files changed, 67652 insertions(+), 22750 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/42216997/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java ---------------------------------------------------------------------- diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index 27f68df..736a22d 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -2187,6 +2187,13 @@ public class HiveConf extends Configuration { "Channel logging level for remote Spark driver. One of {DEBUG, ERROR, INFO, TRACE, WARN}."), SPARK_RPC_SASL_MECHANISM("hive.spark.client.rpc.sasl.mechanisms", "DIGEST-MD5", "Name of the SASL mechanism to use for authentication."), + SPARK_DYNAMIC_PARTITION_PRUNING( + "hive.spark.dynamic.partition.pruning", false, + "When dynamic pruning is enabled, joins on partition keys will be processed by writing\n" + + "to a temporary HDFS file, and read later for removing unnecessary partitions."), + SPARK_DYNAMIC_PARTITION_PRUNING_MAX_DATA_SIZE( + "hive.spark.dynamic.partition.pruning.max.data.size", 100*1024*1024L, + "Maximum total data size in dynamic pruning."), NWAYJOINREORDER("hive.reorder.nway.joins", true, "Runs reordering of tables within single n-way join (i.e.: picks streamtable)"), HIVE_LOG_N_RECORDS("hive.log.every.n.records", 0L, new RangeValidator(0L, null), http://git-wip-us.apache.org/repos/asf/hive/blob/42216997/itests/qtest/pom.xml ---------------------------------------------------------------------- diff --git a/itests/qtest/pom.xml b/itests/qtest/pom.xml index e195bee..44d30da 100644 --- a/itests/qtest/pom.xml +++ b/itests/qtest/pom.xml @@ -429,7 +429,7 @@ templatePath="${basedir}/${hive.path.to.root}/ql/src/test/templates/" template="TestCliDriver.vm" queryDirectory="${basedir}/${hive.path.to.root}/ql/src/test/queries/clientpositive/" queryFile="${qfile}" - excludeQueryFile="${minimr.query.files},${minitez.query.files},${encrypted.query.files}" + excludeQueryFile="${minimr.query.files},${minitez.query.files},${encrypted.query.files},${spark.only.query.files}" queryFileRegex="${qfile_regex}" clusterMode="${clustermode}" runDisabled="${run_disabled}" http://git-wip-us.apache.org/repos/asf/hive/blob/42216997/itests/src/test/resources/testconfiguration.properties ---------------------------------------------------------------------- diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties index f4ae684..2cf4920 100644 --- a/itests/src/test/resources/testconfiguration.properties +++ b/itests/src/test/resources/testconfiguration.properties @@ -1122,6 +1122,12 @@ spark.query.files=add_part_multiple.q, \ vectorized_timestamp_funcs.q, \ windowing.q +# Unlike "spark.query.files" above, these tests only run +# under Spark engine. +spark.only.query.files=spark_dynamic_partition_pruning.q,\ + spark_dynamic_partition_pruning_2.q,\ + spark_vectorized_dynamic_partition_pruning.q + miniSparkOnYarn.query.files=auto_sortmerge_join_16.q,\ bucket4.q,\ bucket5.q,\
