This is an automated email from the ASF dual-hosted git repository. abstractdog pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/hive.git.
from eff0edd HIVE-25132: HiveOperationType should be ReadDatabases for default policy in ranger (#2294) (Sai Hemanth Gantasala, reviewed by Yongzhi Chen) add 9ff9653 HIVE-24761: Support vectorization for bounded windows in PTF (#2099) (Laszlo Bodor reviewed by Ramesh Kumar Thangarajan) No new revisions were added by this update. Summary of changes: .../java/org/apache/hadoop/hive/conf/HiveConf.java | 8 +- .../files/vector_ptf_part_simple_all_datatypes.txt | 40 + .../ExpressionTemplates/ColumnArithmeticColumn.txt | 17 +- .../ColumnArithmeticColumnDecimal.txt | 17 +- .../ExpressionTemplates/ColumnArithmeticScalar.txt | 9 +- .../ColumnArithmeticScalarDecimal.txt | 9 +- .../ExpressionTemplates/ColumnBetween.txt | 10 +- .../ExpressionTemplates/ColumnCompareColumn.txt | 17 +- .../ExpressionTemplates/ColumnCompareScalar.txt | 9 +- .../ExpressionTemplates/ColumnDivideColumn.txt | 17 +- .../ColumnDivideColumnDecimal.txt | 17 +- .../ExpressionTemplates/ColumnDivideScalar.txt | 9 +- .../ColumnDivideScalarDecimal.txt | 9 +- .../ExpressionTemplates/ColumnUnaryFunc.txt | 12 +- .../ExpressionTemplates/ColumnUnaryMinus.txt | 12 +- ...DateColumnArithmeticIntervalYearMonthColumn.txt | 17 +- ...DateColumnArithmeticIntervalYearMonthScalar.txt | 9 +- .../DateColumnArithmeticTimestampColumn.txt | 17 +- .../DateColumnArithmeticTimestampScalar.txt | 9 +- ...DateScalarArithmeticIntervalYearMonthColumn.txt | 9 +- .../DateScalarArithmeticTimestampColumn.txt | 9 +- .../Decimal64ColumnArithmeticDecimal64Column.txt | 17 +- .../Decimal64ColumnArithmeticDecimal64Scalar.txt | 9 +- ...al64ColumnArithmeticDecimal64ScalarUnscaled.txt | 2 +- .../ExpressionTemplates/Decimal64ColumnBetween.txt | 2 +- .../Decimal64ColumnCompareDecimal64Scalar.txt | 2 +- .../Decimal64ColumnDivideDecimal64Column.txt | 17 +- .../Decimal64ColumnDivideDecimal64Scalar.txt | 9 +- .../ExpressionTemplates/Decimal64ColumnScaleUp.txt | 2 +- .../Decimal64ScalarArithmeticDecimal64Column.txt | 9 +- ...al64ScalarArithmeticDecimal64ColumnUnscaled.txt | 2 +- .../Decimal64ScalarCompareDecimal64Column.txt | 2 +- .../ExpressionTemplates/DecimalColumnBetween.txt | 10 +- .../DecimalColumnCompareDecimalColumn.txt | 17 +- .../DecimalColumnCompareDecimalScalar.txt | 9 +- .../ExpressionTemplates/DecimalColumnUnaryFunc.txt | 12 +- .../DecimalScalarCompareDecimalColumn.txt | 9 +- .../ExpressionTemplates/FilterColumnBetween.txt | 12 +- .../FilterColumnCompareColumn.txt | 21 +- .../FilterColumnCompareScalar.txt | 8 +- .../FilterDecimalColumnBetween.txt | 12 +- .../FilterDecimalColumnCompareDecimalColumn.txt | 17 +- .../FilterDecimalColumnCompareDecimalScalar.txt | 9 +- .../FilterDecimalScalarCompareDecimalColumn.txt | 9 +- ...ilterLongDoubleColumnCompareTimestampColumn.txt | 21 +- ...ilterLongDoubleScalarCompareTimestampColumn.txt | 9 +- .../FilterScalarCompareColumn.txt | 9 +- .../FilterStringColumnBetween.txt | 12 +- ...erStringGroupColumnCompareStringGroupColumn.txt | 17 +- ...ringGroupColumnCompareStringGroupScalarBase.txt | 9 +- ...ringGroupScalarCompareStringGroupColumnBase.txt | 9 +- .../FilterTimestampColumnBetween.txt | 12 +- ...ilterTimestampColumnCompareLongDoubleColumn.txt | 21 +- ...ilterTimestampColumnCompareLongDoubleScalar.txt | 9 +- ...FilterTimestampColumnCompareTimestampColumn.txt | 17 +- ...FilterTimestampColumnCompareTimestampScalar.txt | 9 +- ...ilterTimestampScalarCompareLongDoubleColumn.txt | 4 +- ...FilterTimestampScalarCompareTimestampColumn.txt | 9 +- .../FilterTruncStringColumnBetween.txt | 12 +- .../ExpressionTemplates/IfExprColumnScalar.txt | 14 +- .../IfExprObjectColumnColumn.txt | 24 +- .../IfExprObjectColumnScalar.txt | 14 +- .../IfExprObjectScalarColumn.txt | 16 +- .../IfExprObjectScalarScalar.txt | 9 +- .../ExpressionTemplates/IfExprScalarColumn.txt | 16 +- .../ExpressionTemplates/IfExprScalarScalar.txt | 9 +- ...IntervalYearMonthColumnArithmeticDateColumn.txt | 17 +- ...IntervalYearMonthColumnArithmeticDateScalar.txt | 9 +- ...valYearMonthColumnArithmeticTimestampColumn.txt | 17 +- ...valYearMonthColumnArithmeticTimestampScalar.txt | 9 +- ...IntervalYearMonthScalarArithmeticDateColumn.txt | 9 +- ...valYearMonthScalarArithmeticTimestampColumn.txt | 9 +- .../LongDoubleColumnCompareTimestampColumn.txt | 17 +- .../LongDoubleColumnCompareTimestampScalar.txt | 9 +- .../LongDoubleScalarCompareTimestampColumn.txt | 9 +- .../ExpressionTemplates/ScalarArithmeticColumn.txt | 9 +- .../ScalarArithmeticColumnDecimal.txt | 9 +- .../ExpressionTemplates/ScalarCompareColumn.txt | 9 +- .../ExpressionTemplates/ScalarDivideColumn.txt | 9 +- .../ScalarDivideColumnDecimal.txt | 9 +- .../ExpressionTemplates/StringColumnBetween.txt | 10 +- .../StringGroupColumnCompareStringGroupColumn.txt | 17 +- ...ringGroupColumnCompareStringGroupScalarBase.txt | 9 +- .../StringGroupColumnCompareTruncStringScalar.txt | 2 +- ...ringGroupScalarCompareStringGroupColumnBase.txt | 9 +- .../TimestampColumnArithmeticDateColumn.txt | 17 +- .../TimestampColumnArithmeticDateScalar.txt | 9 +- ...tampColumnArithmeticIntervalYearMonthColumn.txt | 17 +- ...tampColumnArithmeticIntervalYearMonthScalar.txt | 9 +- .../TimestampColumnArithmeticTimestampColumn.txt | 17 +- .../TimestampColumnArithmeticTimestampScalar.txt | 9 +- .../ExpressionTemplates/TimestampColumnBetween.txt | 10 +- .../TimestampColumnCompareLongDoubleColumn.txt | 9 +- .../TimestampColumnCompareLongDoubleScalar.txt | 9 +- .../TimestampColumnCompareTimestampColumn.txt | 9 +- .../TimestampColumnCompareTimestampScalar.txt | 9 +- .../TimestampScalarArithmeticDateColumn.txt | 9 +- ...tampScalarArithmeticIntervalYearMonthColumn.txt | 9 +- .../TimestampScalarArithmeticTimestampColumn.txt | 11 +- .../TimestampScalarCompareTimestampColumn.txt | 9 +- .../apache/hadoop/hive/ql/exec/PTFPartition.java | 30 +- .../apache/hadoop/hive/ql/exec/PTFValueCache.java | 164 + .../hive/ql/exec/vector/VectorizedBatchUtil.java | 28 +- .../vector/expressions/BucketNumExpression.java | 2 +- .../CastBooleanToCharViaLongToChar.java | 2 +- .../CastBooleanToVarCharViaLongToVarChar.java | 2 +- .../exec/vector/expressions/CastCharToBinary.java | 2 +- .../ql/exec/vector/expressions/CastDateToChar.java | 2 +- .../vector/expressions/CastDateToTimestamp.java | 12 +- .../exec/vector/expressions/CastDateToVarChar.java | 2 +- .../exec/vector/expressions/CastDecimalToChar.java | 2 +- .../vector/expressions/CastDecimalToDecimal.java | 12 +- .../vector/expressions/CastDecimalToVarChar.java | 2 +- .../exec/vector/expressions/CastDoubleToChar.java | 2 +- .../vector/expressions/CastDoubleToTimestamp.java | 9 +- .../vector/expressions/CastDoubleToVarChar.java | 2 +- .../exec/vector/expressions/CastFloatToChar.java | 2 +- .../vector/expressions/CastFloatToVarChar.java | 2 +- .../ql/exec/vector/expressions/CastLongToChar.java | 2 +- .../vector/expressions/CastLongToDecimal64.java | 11 +- .../vector/expressions/CastLongToTimestamp.java | 12 +- .../exec/vector/expressions/CastLongToVarChar.java | 2 +- .../CastMillisecondsLongToTimestamp.java | 9 +- .../vector/expressions/CastStringGroupToChar.java | 2 +- .../expressions/CastStringGroupToVarChar.java | 2 +- .../exec/vector/expressions/CastStringToDate.java | 12 +- .../vector/expressions/CastStringToDecimal.java | 12 +- .../vector/expressions/CastStringToDouble.java | 11 +- .../expressions/CastStringToIntervalDayTime.java | 12 +- .../expressions/CastStringToIntervalYearMonth.java | 12 +- .../exec/vector/expressions/CastStringToLong.java | 11 +- .../vector/expressions/CastStringToTimestamp.java | 14 +- .../vector/expressions/CastTimestampToBoolean.java | 12 +- .../vector/expressions/CastTimestampToChar.java | 2 +- .../vector/expressions/CastTimestampToDouble.java | 12 +- .../vector/expressions/CastTimestampToLong.java | 9 +- .../vector/expressions/CastTimestampToVarChar.java | 2 +- .../hive/ql/exec/vector/expressions/ColAndCol.java | 18 +- .../hive/ql/exec/vector/expressions/ColOrCol.java | 18 +- .../expressions/ConstantVectorExpression.java | 2 +- .../expressions/DateColSubtractDateColumn.java | 17 +- .../expressions/DateColSubtractDateScalar.java | 9 +- .../expressions/DateScalarSubtractDateColumn.java | 9 +- .../vector/expressions/Decimal64ColumnInList.java | 2 +- .../vector/expressions/DecimalColumnInList.java | 11 +- .../expressions/DecimalToStringUnaryUDF.java | 11 +- .../vector/expressions/DoubleColumnInList.java | 8 +- .../vector/expressions/DoubleToStringUnaryUDF.java | 12 +- .../expressions/DynamicValueVectorExpression.java | 2 +- .../expressions/FilterDecimal64ColumnBetween.java | 2 +- .../FilterDecimal64ColumnNotBetween.java | 2 +- .../expressions/FilterTimestampColumnInList.java | 10 +- .../ql/exec/vector/expressions/FuncDateToDate.java | 11 +- .../vector/expressions/FuncDecimalToDouble.java | 11 +- .../exec/vector/expressions/FuncDecimalToLong.java | 11 +- .../vector/expressions/FuncDecimalToTimestamp.java | 11 +- .../vector/expressions/FuncDoubleToDecimal.java | 11 +- .../exec/vector/expressions/FuncLongToDecimal.java | 11 +- .../exec/vector/expressions/FuncLongToString.java | 12 +- .../hive/ql/exec/vector/expressions/FuncRand.java | 2 +- .../ql/exec/vector/expressions/FuncRandNoSeed.java | 2 +- .../FuncRoundWithNumDigitsDecimalToDecimal.java | 11 +- .../exec/vector/expressions/FuncStringToLong.java | 21 +- .../vector/expressions/FuncTimestampToDecimal.java | 12 +- .../vector/expressions/FuncTimestampToLong.java | 12 +- .../expressions/FuncTimestampToTimestamp.java | 11 +- .../ql/exec/vector/expressions/GroupingColumn.java | 2 +- .../exec/vector/expressions/GroupingColumns.java | 2 +- .../vector/expressions/IdentityExpression.java | 2 +- .../vector/expressions/IfExprColumnCondExpr.java | 19 +- .../exec/vector/expressions/IfExprColumnNull.java | 17 +- .../vector/expressions/IfExprCondExprBase.java | 19 +- .../vector/expressions/IfExprCondExprColumn.java | 19 +- .../vector/expressions/IfExprCondExprCondExpr.java | 19 +- .../vector/expressions/IfExprCondExprNull.java | 12 +- .../IfExprDecimal64ColumnDecimal64Scalar.java | 2 +- .../IfExprDecimal64ScalarDecimal64Column.java | 4 +- .../IfExprDecimal64ScalarDecimal64Scalar.java | 2 +- .../IfExprDoubleColumnDoubleColumn.java | 24 +- .../expressions/IfExprLongColumnLongColumn.java | 24 +- .../exec/vector/expressions/IfExprNullColumn.java | 17 +- .../vector/expressions/IfExprNullCondExpr.java | 12 +- .../ql/exec/vector/expressions/IfExprNullNull.java | 2 +- .../IfExprStringGroupColumnStringGroupColumn.java | 24 +- .../IfExprStringGroupColumnStringScalar.java | 16 +- .../IfExprStringScalarStringGroupColumn.java | 14 +- .../IfExprStringScalarStringScalar.java | 9 +- .../hive/ql/exec/vector/expressions/IsNotNull.java | 12 +- .../hive/ql/exec/vector/expressions/IsNull.java | 12 +- .../vector/expressions/ListIndexColColumn.java | 13 +- .../vector/expressions/ListIndexColScalar.java | 8 +- .../expressions/LongColDivideLongColumn.java | 17 +- .../expressions/LongColDivideLongScalar.java | 9 +- .../exec/vector/expressions/LongColumnInList.java | 8 +- .../expressions/LongScalarDivideLongColumn.java | 9 +- .../vector/expressions/LongToStringUnaryUDF.java | 12 +- .../vector/expressions/MathFuncDoubleToDouble.java | 12 +- .../vector/expressions/MathFuncLongToDouble.java | 12 +- .../vector/expressions/MathFuncLongToLong.java | 12 +- .../vector/expressions/MurmurHashExpression.java | 12 +- .../hive/ql/exec/vector/expressions/NotCol.java | 12 +- .../vector/expressions/NullVectorExpression.java | 2 +- .../ql/exec/vector/expressions/OctetLength.java | 12 +- .../vector/expressions/PosModDoubleToDouble.java | 2 +- .../exec/vector/expressions/PosModLongToLong.java | 2 +- .../RoundWithNumDigitsDoubleToDouble.java | 2 +- .../exec/vector/expressions/ScalarNullAndCol.java | 7 +- .../exec/vector/expressions/ScalarNullOrCol.java | 7 +- .../SelectStringColLikeStringScalar.java | 12 +- .../vector/expressions/StringColumnInList.java | 8 +- .../StringGroupColConcatStringScalar.java | 9 +- .../expressions/StringGroupConcatColCol.java | 17 +- .../ql/exec/vector/expressions/StringLength.java | 12 +- .../StringScalarConcatStringGroupCol.java | 9 +- .../vector/expressions/StringSubstrColStart.java | 10 +- .../expressions/StringSubstrColStartLen.java | 10 +- .../ql/exec/vector/expressions/StringUnaryUDF.java | 9 +- .../vector/expressions/StringUnaryUDFDirect.java | 12 +- .../vector/expressions/StructColumnInList.java | 2 +- .../vector/expressions/TimestampColumnInList.java | 12 +- .../expressions/TimestampToStringUnaryUDF.java | 12 +- .../vector/expressions/TruncDateFromTimestamp.java | 17 +- .../vector/expressions/TruncDecimalNoScale.java | 1 - .../ql/exec/vector/expressions/TruncFloat.java | 9 +- .../exec/vector/expressions/TruncFloatNoScale.java | 1 - .../ql/exec/vector/expressions/VectorCoalesce.java | 19 +- .../hive/ql/exec/vector/expressions/VectorElt.java | 23 +- .../exec/vector/expressions/VectorExpression.java | 63 +- .../VectorInBloomFilterColDynamicValue.java | 7 +- .../vector/expressions/VectorUDFDateAddColCol.java | 17 +- .../expressions/VectorUDFDateAddColScalar.java | 9 +- .../expressions/VectorUDFDateAddScalarCol.java | 12 +- .../expressions/VectorUDFDateDiffColCol.java | 9 +- .../expressions/VectorUDFDateDiffColScalar.java | 12 +- .../expressions/VectorUDFDateDiffScalarCol.java | 12 +- .../expressions/VectorUDFMapIndexBaseCol.java | 14 +- .../expressions/VectorUDFMapIndexBaseScalar.java | 9 +- .../vector/expressions/VectorUDFStructField.java | 8 +- .../expressions/VectorUDFTimestampFieldDate.java | 11 +- .../expressions/VectorUDFTimestampFieldString.java | 10 +- .../VectorUDFTimestampFieldTimestamp.java | 11 +- .../vector/ptf/BufferedVectorizedRowBatch.java} | 33 +- .../vector/ptf/VectorPTFEvaluatorAbstractAvg.java | 103 + .../vector/ptf/VectorPTFEvaluatorAbstractSum.java | 94 + .../ql/exec/vector/ptf/VectorPTFEvaluatorBase.java | 89 +- .../exec/vector/ptf/VectorPTFEvaluatorCount.java | 2 +- .../ptf/VectorPTFEvaluatorCountDistinct.java | 4 +- .../vector/ptf/VectorPTFEvaluatorCountStar.java | 16 +- .../vector/ptf/VectorPTFEvaluatorDecimalAvg.java | 39 +- .../ptf/VectorPTFEvaluatorDecimalFirstValue.java | 16 + .../ptf/VectorPTFEvaluatorDecimalLastValue.java | 6 +- .../vector/ptf/VectorPTFEvaluatorDecimalMax.java | 2 +- .../vector/ptf/VectorPTFEvaluatorDecimalMin.java | 3 +- .../vector/ptf/VectorPTFEvaluatorDecimalSum.java | 35 +- .../vector/ptf/VectorPTFEvaluatorDenseRank.java | 15 +- .../vector/ptf/VectorPTFEvaluatorDoubleAvg.java | 35 +- .../ptf/VectorPTFEvaluatorDoubleFirstValue.java | 16 + .../ptf/VectorPTFEvaluatorDoubleLastValue.java | 6 +- .../vector/ptf/VectorPTFEvaluatorDoubleMax.java | 2 +- .../vector/ptf/VectorPTFEvaluatorDoubleMin.java | 2 +- .../vector/ptf/VectorPTFEvaluatorDoubleSum.java | 22 +- .../exec/vector/ptf/VectorPTFEvaluatorHelper.java | 93 + .../exec/vector/ptf/VectorPTFEvaluatorLongAvg.java | 37 +- .../ptf/VectorPTFEvaluatorLongFirstValue.java | 16 + .../ptf/VectorPTFEvaluatorLongLastValue.java | 6 +- .../exec/vector/ptf/VectorPTFEvaluatorLongMax.java | 2 +- .../exec/vector/ptf/VectorPTFEvaluatorLongMin.java | 2 +- .../exec/vector/ptf/VectorPTFEvaluatorLongSum.java | 24 +- .../ql/exec/vector/ptf/VectorPTFEvaluatorRank.java | 13 +- .../vector/ptf/VectorPTFEvaluatorRowNumber.java | 6 +- .../ql/exec/vector/ptf/VectorPTFGroupBatches.java | 1249 ++++- .../hive/ql/exec/vector/ptf/VectorPTFOperator.java | 403 +- .../exec/vector/ptf/VectorSpillBlockContainer.java | 407 ++ .../hive/ql/exec/vector/udf/VectorUDFAdaptor.java | 2 +- .../hive/ql/optimizer/physical/Vectorizer.java | 48 +- .../org/apache/hadoop/hive/ql/plan/PTFDesc.java | 13 + .../apache/hadoop/hive/ql/plan/VectorPTFDesc.java | 36 +- .../hive/ql/udf/ptf/BasePartitionEvaluator.java | 104 +- .../hadoop/hive/ql/udf/ptf/PTFRangeUtil.java | 90 + .../org/apache/hadoop/hive/ql/udf/ptf/Range.java | 111 + .../hive/ql/udf/ptf/ValueBoundaryScanner.java | 330 +- .../ql/exec/vector/mapjoin/NoOpExpression.java | 2 +- .../exec/vector/ptf/TestVectorPTFGroupBatches.java | 661 +++ .../hadoop/hive/ql/udf/ptf/TestPTFValueCache.java | 86 + .../apache/hadoop/hive/ql/udf/ptf/TestRange.java | 77 + .../clientpositive/vector_ptf_bounded_start.q | 603 +++ ql/src/test/results/clientpositive/llap/ptf.q.out | 4 +- .../llap/vector_groupby_grouping_window.q.out | 1 + .../llap/vector_outer_reference_windowed.q.out | 20 +- .../llap/vector_ptf_bounded_start.q.out | 5567 ++++++++++++++++++++ .../llap/vector_ptf_count_distinct.q.out | 48 + .../llap/vector_ptf_last_value_streaming.q.out | 1 + .../llap/vector_ptf_part_simple.q.out | 44 +- .../clientpositive/llap/vector_windowing.q.out | 286 +- .../llap/vector_windowing_expressions.q.out | 4 + .../clientpositive/llap/vector_windowing_gby.q.out | 1 + .../llap/vector_windowing_gby2.q.out | 5 + .../llap/vector_windowing_multipartitioning.q.out | 14 + .../llap/vector_windowing_navfn.q.out | 17 +- .../llap/vector_windowing_order_null.q.out | 10 +- .../llap/vector_windowing_range_multiorder.q.out | 9 +- .../llap/vector_windowing_rank.q.out | 6 + .../llap/vector_windowing_streaming.q.out | 4 + .../llap/vector_windowing_windowspec.q.out | 15 +- .../clientpositive/llap/vectorized_ptf.q.out | 101 +- .../clientpositive/llap/windowing_decimal.q.out | 40 +- .../clientpositive/llap/windowing_udaf.q.out | 2 +- .../apache/hadoop/hive/tools/GenVectorCode.java | 4 +- 308 files changed, 11476 insertions(+), 2413 deletions(-) create mode 100644 data/files/vector_ptf_part_simple_all_datatypes.txt create mode 100644 ql/src/java/org/apache/hadoop/hive/ql/exec/PTFValueCache.java copy ql/src/{test/org/apache/hadoop/hive/ql/exec/vector/mapjoin/NoOpExpression.java => java/org/apache/hadoop/hive/ql/exec/vector/ptf/BufferedVectorizedRowBatch.java} (50%) create mode 100644 ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorAbstractAvg.java create mode 100644 ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorAbstractSum.java create mode 100644 ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorHelper.java create mode 100644 ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorSpillBlockContainer.java create mode 100644 ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/PTFRangeUtil.java create mode 100644 ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/Range.java create mode 100644 ql/src/test/org/apache/hadoop/hive/ql/exec/vector/ptf/TestVectorPTFGroupBatches.java create mode 100644 ql/src/test/org/apache/hadoop/hive/ql/udf/ptf/TestPTFValueCache.java create mode 100644 ql/src/test/org/apache/hadoop/hive/ql/udf/ptf/TestRange.java create mode 100644 ql/src/test/queries/clientpositive/vector_ptf_bounded_start.q create mode 100644 ql/src/test/results/clientpositive/llap/vector_ptf_bounded_start.q.out