ASTERIXDB-1228: Add MISSING into the data model. 1. MISSING repsents the value of a non-existing field in a record or an out-of-bound index access of a collection; 2. NULL represents that the value of an optional field in a record is unknown or the value of existing collection entry is unknown. 3. Unit tests for all missing/null-in-missing/null-out scalar functions.
Change-Id: Ia49ed8474bfc5d6604231819065117468c5b0897 Reviewed-on: https://asterix-gerrit.ics.uci.edu/846 Tested-by: Jenkins <[email protected]> Reviewed-by: Till Westmann <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/commit/535d86b5 Tree: http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/tree/535d86b5 Diff: http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/diff/535d86b5 Branch: refs/heads/master Commit: 535d86b56d3cd525b3f36a2b56143d8f7dd075c5 Parents: 09d4f9f Author: Yingyi Bu <[email protected]> Authored: Mon May 23 16:44:25 2016 -0700 Committer: Yingyi Bu <[email protected]> Committed: Mon May 23 18:31:06 2016 -0700 ---------------------------------------------------------------------- .../jobgen/QueryLogicalExpressionJobGen.java | 29 +- .../rules/ByNameToByIndexFieldAccessRule.java | 68 ++- .../rules/CheckFilterExpressionTypeRule.java | 4 +- .../optimizer/rules/ConstantFoldingRule.java | 12 +- .../asterix/optimizer/rules/FuzzyJoinRule.java | 6 +- .../rules/InlineUnnestFunctionRule.java | 24 +- .../rules/IntroduceAutogenerateIDRule.java | 2 +- ...eDynamicTypeCastForExternalFunctionRule.java | 12 +- .../rules/IntroduceDynamicTypeCastRule.java | 22 +- .../rules/IntroduceEnforcedListTypeRule.java | 4 +- ...IntroduceSecondaryIndexInsertDeleteRule.java | 14 +- .../IntroduceStaticTypeCastForInsertRule.java | 4 +- .../rules/ReplaceSinkOpWithCommitOpRule.java | 18 +- .../rules/SetClosedRecordConstructorsRule.java | 53 +- .../am/AbstractIntroduceAccessMethodRule.java | 2 +- .../optimizer/rules/am/AccessMethodUtils.java | 4 +- .../rules/am/IntroduceJoinAccessMethodRule.java | 2 +- .../rules/am/InvertedIndexAccessMethod.java | 8 +- ...ineSubplanInputForNestedTupleSourceRule.java | 6 +- .../rules/typecast/StaticTypeCastUtil.java | 70 +-- .../asterix/translator/ConstantHelper.java | 34 +- .../LangExpressionToPlanTranslator.java | 1 + .../asterix/translator/TypeTranslator.java | 26 +- .../translator/util/FunctionCollection.java | 495 ++++++++++--------- .../asterix/translator/util/ValidateUtil.java | 4 +- .../apache/asterix/api/common/APIFramework.java | 12 +- .../file/SecondaryIndexOperationsHelper.java | 6 +- .../apache/asterix/runtime/NullMissingTest.java | 105 ++++ .../metadata_datatype/metadata_datatype.1.adm | 3 +- .../has-param1/has-param1.3.query.aql | 4 +- .../null-missing/array/array.1.query.sqlpp | 20 + .../null-missing/boolean/boolean.1.query.sqlpp | 37 ++ .../null-missing/ceil/ceil.1.query.sqlpp | 23 + .../field-access/field-access.1.query.sqlpp | 23 + .../null-missing/length/length.1.query.sqlpp | 23 + .../null-missing/lower/lower.1.query.sqlpp | 23 + .../scan-collection.1.query.sqlpp | 24 + .../null-missing/substr/substr.1.query.sqlpp | 43 ++ .../null-missing/upper/upper.1.query.sqlpp | 23 + .../queries_sqlpp/null/ceil/ceil.1.query.sqlpp | 20 - .../null/length/length.1.query.sqlpp | 20 - .../null/lower/lower.1.query.sqlpp | 20 - .../null/substr/substr.1.query.sqlpp | 31 -- .../null/upper/upper.1.query.sqlpp | 20 - .../has-param1/has-param1.3.query.sqlpp | 2 +- .../tinysocial-suite.12.query.sqlpp | 2 +- .../tinysocial-suite.13.query.sqlpp | 2 +- .../results/custord/order_q_04/order_q_04.1.adm | 8 +- .../results/custord/order_q_05/order_q_05.1.adm | 8 +- .../results/null-missing/array/array.1.adm | 1 + .../results/null-missing/boolean/boolean.1.adm | 1 + .../results/null-missing/ceil/ceil.1.adm | 1 + .../field-access/field-access.1.adm | 1 + .../results/null-missing/length/length.1.adm | 1 + .../results/null-missing/lower/lower.1.adm | 1 + .../scan-collection/scan-collection.1.adm | 1 + .../results/null-missing/substr/substr.1.adm | 1 + .../results/null-missing/upper/upper.1.adm | 1 + .../runtimets/results/null/ceil/ceil.1.adm | 1 - .../runtimets/results/null/length/length.1.adm | 1 - .../runtimets/results/null/lower/lower.1.adm | 1 - .../runtimets/results/null/substr/substr.1.adm | 1 - .../runtimets/results/null/upper/upper.1.adm | 1 - .../query-ASTERIXDB-1329.24.adm | 18 +- .../opentype_orderby_01.1.adm | 20 +- .../semistructured/has-param1/has-param1.3.ast | 2 +- .../tinysocial-suite.12.ast | 2 +- .../tinysocial-suite.13.ast | 2 +- .../resources/runtimets/testsuite_sqlpp.xml | 32 +- ...ixLSMTreeInsertDeleteOperatorDescriptor.java | 5 +- .../adapter/factory/LookupAdapterFactory.java | 16 +- .../external/dataset/adapter/LookupAdapter.java | 22 +- .../external/indexing/RecordIdReader.java | 4 +- .../library/ExternalFunctionProvider.java | 8 +- .../external/library/JTypeObjectFactory.java | 15 +- .../external/library/java/JObjectUtil.java | 36 +- .../asterix/external/library/java/JObjects.java | 2 +- .../ExternalBTreeSearchOperatorDescriptor.java | 8 +- ...ExternalBTreeSearchOperatorNodePushable.java | 12 +- .../ExternalLookupOperatorDescriptor.java | 10 +- .../ExternalRTreeSearchOperatorDescriptor.java | 8 +- ...ExternalRTreeSearchOperatorNodePushable.java | 12 +- .../operators/IndexInfoOperatorDescriptor.java | 6 +- .../asterix/external/parser/ADMDataParser.java | 30 +- .../external/parser/DelimitedDataParser.java | 4 +- .../external/parser/HiveRecordParser.java | 2 +- .../parser/RecordWithMetadataParser.java | 4 +- .../provider/AdapterFactoryProvider.java | 8 +- .../external/util/ExternalDataUtils.java | 8 +- .../asterix/external/library/ClassAdParser.java | 21 +- .../asterix-lang-aql/src/main/javacc/AQL.jj | 8 +- .../asterix/lang/common/base/Literal.java | 1 + .../lang/common/literal/MissingLiteral.java | 55 +++ .../lang/common/visitor/FormatPrintVisitor.java | 7 +- .../lang/common/visitor/QueryPrintVisitor.java | 2 +- .../asterix-lang-sqlpp/src/main/javacc/SQLPP.jj | 8 +- .../plugin/EvaluatorGeneratorMojo.java | 1 + .../metadata/bootstrap/MetadataRecordTypes.java | 18 +- .../entities/AsterixBuiltinTypeMap.java | 4 +- .../apache/asterix/metadata/entities/Index.java | 5 +- .../DatatypeTupleTranslator.java | 7 +- .../asterix/builders/AbstractListBuilder.java | 28 +- .../apache/asterix/builders/RecordBuilder.java | 29 +- .../data/common/AqlExpressionTypeComputer.java | 4 +- .../data/common/AqlMissableTypeComputer.java | 52 ++ .../data/common/AqlNullableTypeComputer.java | 55 --- .../AqlPartialAggregationTypeComputer.java | 18 +- .../data/nontagged/AqlMissingWriterFactory.java | 50 ++ .../data/nontagged/AqlNullWriterFactory.java | 53 -- .../comparators/ABinaryComparator.java | 24 - .../AObjectAscBinaryComparatorFactory.java | 23 +- .../ListItemBinaryComparatorFactory.java | 12 +- .../adm/ANullableFieldPrinterFactory.java | 4 +- .../nontagged/printers/adm/AObjectPrinter.java | 1 + .../csv/ANullableFieldPrinterFactory.java | 6 +- .../nontagged/printers/csv/AObjectPrinter.java | 1 + .../clean/ANullableFieldPrinterFactory.java | 6 +- .../printers/json/clean/AObjectPrinter.java | 1 + .../lossless/ANullableFieldPrinterFactory.java | 6 +- .../printers/json/lossless/AObjectPrinter.java | 1 + .../serde/AMissingSerializerDeserializer.java | 47 ++ .../serde/ANullSerializerDeserializer.java | 6 +- .../serde/AObjectSerializerDeserializer.java | 9 +- .../serde/ARecordSerializerDeserializer.java | 9 +- .../asterix/formats/base/IDataFormat.java | 4 +- .../nontagged/AqlADMPrinterFactoryProvider.java | 7 +- .../AqlBinaryBooleanInspectorImpl.java | 7 +- .../AqlBinaryComparatorFactoryProvider.java | 4 +- .../nontagged/AqlCSVPrinterFactoryProvider.java | 7 +- .../AqlCleanJSONPrinterFactoryProvider.java | 7 +- .../AqlLosslessJSONPrinterFactoryProvider.java | 7 +- .../AqlPredicateEvaluatorFactoryProvider.java | 15 +- .../AqlSerializerDeserializerProvider.java | 10 +- .../org/apache/asterix/om/base/AMissing.java | 69 +++ .../java/org/apache/asterix/om/base/ANull.java | 11 +- .../om/constants/AsterixConstantValue.java | 6 + .../om/functions/AsterixBuiltinFunctions.java | 483 +++++++++--------- .../pointables/ARecordVisitablePointable.java | 20 +- .../om/pointables/cast/ARecordCaster.java | 62 ++- .../pointables/nonvisitor/ARecordPointable.java | 2 +- .../om/pointables/printer/adm/AListPrinter.java | 8 +- .../pointables/printer/adm/APrintVisitor.java | 11 +- .../pointables/printer/adm/ARecordPrinter.java | 16 +- .../pointables/printer/csv/APrintVisitor.java | 11 +- .../pointables/printer/csv/ARecordPrinter.java | 8 +- .../printer/json/clean/AListPrinter.java | 8 +- .../printer/json/clean/APrintVisitor.java | 11 +- .../printer/json/clean/ARecordPrinter.java | 8 +- .../printer/json/lossless/AListPrinter.java | 8 +- .../printer/json/lossless/APrintVisitor.java | 11 +- .../printer/json/lossless/ARecordPrinter.java | 8 +- .../base/AbstractResultTypeComputer.java | 64 +++ .../om/typecomputer/base/TypeCastUtils.java | 57 +++ .../base/TypeComputerUtilities.java | 124 ----- .../typecomputer/impl/ABinaryTypeComputer.java | 12 +- .../typecomputer/impl/ABooleanTypeComputer.java | 11 +- .../typecomputer/impl/ACircleTypeComputer.java | 11 +- .../impl/ADateTimeTypeComputer.java | 11 +- .../om/typecomputer/impl/ADateTypeComputer.java | 11 +- .../impl/ADayTimeDurationTypeComputer.java | 38 ++ .../typecomputer/impl/ADoubleTypeComputer.java | 11 +- .../impl/ADurationTypeComputer.java | 37 ++ .../typecomputer/impl/AFloatTypeComputer.java | 10 +- .../typecomputer/impl/AInt16TypeComputer.java | 37 ++ .../typecomputer/impl/AInt32TypeComputer.java | 11 +- .../typecomputer/impl/AInt64TypeComputer.java | 11 +- .../om/typecomputer/impl/AInt8TypeComputer.java | 37 ++ .../impl/AIntervalTypeComputer.java | 14 +- .../om/typecomputer/impl/ALineTypeComputer.java | 11 +- .../typecomputer/impl/AMissingTypeComputer.java | 42 ++ .../om/typecomputer/impl/ANullTypeComputer.java | 42 -- .../typecomputer/impl/APoint3DTypeComputer.java | 11 +- .../typecomputer/impl/APointTypeComputer.java | 11 +- .../typecomputer/impl/APolygonTypeComputer.java | 11 +- .../impl/ARectangleTypeComputer.java | 11 +- .../typecomputer/impl/AStringTypeComputer.java | 11 +- .../impl/ATemporalInstanceTypeComputer.java | 52 ++ .../om/typecomputer/impl/ATimeTypeComputer.java | 11 +- .../om/typecomputer/impl/AUUIDTypeComputer.java | 11 +- .../impl/AYearMonthDurationTypeComputer.java | 38 ++ .../impl/AbstractBinaryStringTypeComputer.java | 90 ---- .../impl/AbstractQuadStringTypeComputer.java | 66 --- .../impl/AbstractStringTypeComputer.java | 38 ++ .../impl/AbstractTripleStringTypeComputer.java | 62 --- ...BinaryBooleanOrNullFunctionTypeComputer.java | 61 --- .../BinaryStringBoolOrNullTypeComputer.java | 41 -- .../BinaryStringStringOrNullTypeComputer.java | 43 -- .../impl/BooleanFunctionTypeComputer.java | 48 ++ .../impl/BooleanOnlyTypeComputer.java | 42 ++ .../impl/CastListResultTypeComputer.java | 4 +- .../impl/CastRecordResultTypeComputer.java | 4 +- .../impl/ClosedRecordConstructorResultType.java | 22 +- .../impl/CollectionMemberResultType.java | 54 ++ .../impl/CollectionToSequenceTypeComputer.java | 27 +- .../impl/ConcatNonNullTypeComputer.java | 37 +- .../impl/FieldAccessByIndexResultType.java | 2 +- .../impl/FieldAccessByNameResultType.java | 100 ++++ .../impl/FieldAccessNestedResultType.java | 2 +- .../impl/FlowRecordResultTypeComputer.java | 4 +- .../om/typecomputer/impl/GetItemResultType.java | 54 -- .../GetOverlappingInvervalTypeComputer.java | 12 +- .../impl/InjectFailureTypeComputer.java | 2 +- .../typecomputer/impl/LocalAvgTypeComputer.java | 44 ++ .../impl/MinMaxAggTypeComputer.java | 63 +++ .../NonTaggedCollectionMemberResultType.java | 55 --- .../NonTaggedFieldAccessByNameResultType.java | 104 ---- .../impl/NonTaggedGetItemResultType.java | 47 +- .../impl/NonTaggedLocalAvgTypeComputer.java | 44 -- .../impl/NonTaggedMinMaxAggTypeComputer.java | 106 ---- ...onTaggedNumericAddSubMulDivTypeComputer.java | 343 ------------- .../impl/NonTaggedNumericAggTypeComputer.java | 88 ---- ...ggedNumericRoundHalfToEven2TypeComputer.java | 115 ----- ...nTaggedNumericUnaryFunctionTypeComputer.java | 92 ---- .../impl/NonTaggedSwitchCaseComputer.java | 67 --- .../impl/NonTaggedUnaryMinusTypeComputer.java | 48 -- .../impl/NotMissingTypeComputer.java | 77 +++ .../typecomputer/impl/NotNullTypeComputer.java | 76 --- .../impl/NullableDoubleTypeComputer.java | 42 ++ .../impl/NumericAddSubMulDivTypeComputer.java | 305 ++++++++++++ .../impl/NumericAddSubMulTypeDescriptor.java | 83 ---- .../impl/NumericAggTypeComputer.java | 73 +++ .../NumericRoundHalfToEven2TypeComputer.java | 86 ++++ .../impl/NumericUnaryFunctionTypeComputer.java | 72 +++ .../impl/OpenARecordTypeComputer.java | 43 ++ .../impl/OpenRecordConstructorResultType.java | 4 +- .../impl/OptionalABinaryTypeComputer.java | 48 -- .../impl/OptionalABooleanTypeComputer.java | 48 -- .../impl/OptionalACircleTypeComputer.java | 48 -- .../impl/OptionalADateTimeTypeComputer.java | 48 -- .../impl/OptionalADateTypeComputer.java | 48 -- .../OptionalADayTimeDurationTypeComputer.java | 52 -- .../impl/OptionalADoubleTypeComputer.java | 47 -- .../impl/OptionalADurationTypeComputer.java | 48 -- .../impl/OptionalAFloatTypeComputer.java | 48 -- .../impl/OptionalAInt16TypeComputer.java | 48 -- .../impl/OptionalAInt32TypeComputer.java | 48 -- .../impl/OptionalAInt64TypeComputer.java | 48 -- .../impl/OptionalAInt8TypeComputer.java | 48 -- .../impl/OptionalAIntervalTypeComputer.java | 48 -- .../impl/OptionalALineTypeComputer.java | 48 -- .../impl/OptionalAPoint3DTypeComputer.java | 48 -- .../impl/OptionalAPointTypeComputer.java | 48 -- .../impl/OptionalAPolygonTypeComputer.java | 48 -- .../impl/OptionalARectangleTypeComputer.java | 48 -- .../impl/OptionalAStringTypeComputer.java | 48 -- .../OptionalATemporalInstanceTypeComputer.java | 54 -- .../impl/OptionalATimeTypeComputer.java | 48 -- .../OptionalAYearMonthDurationTypeComputer.java | 52 -- .../impl/OptionalOpenARecordTypeComputer.java | 43 -- .../impl/OrderedListConstructorResultType.java | 31 +- .../impl/OrderedListOfAInt32TypeComputer.java | 11 +- .../impl/OrderedListOfAInt64TypeComputer.java | 11 +- .../OrderedListOfAIntervalTypeComputer.java | 11 +- .../impl/OrderedListOfAPointTypeComputer.java | 11 +- .../impl/OrderedListOfAStringTypeComputer.java | 11 +- .../impl/OrderedListOfAnyTypeComputer.java | 11 +- .../QuadStringStringOrNullTypeComputer.java | 44 -- .../impl/RecordAddFieldsTypeComputer.java | 8 +- .../impl/RecordConstructorResultType.java | 4 +- .../impl/RecordMergeTypeComputer.java | 6 +- .../impl/RecordRemoveFieldsTypeComputer.java | 14 +- .../ScalarVersionOfAggregateResultType.java | 42 +- .../impl/StringBooleanTypeComputer.java | 34 ++ .../impl/StringStringTypeComputer.java | 35 ++ .../impl/SubsetCollectionTypeComputer.java | 4 +- .../impl/Substring2TypeComputer.java | 67 +-- .../impl/SubstringTypeComputer.java | 74 +-- .../typecomputer/impl/SwitchCaseComputer.java | 66 +++ .../TripleStringBoolOrNullTypeComputer.java | 43 -- .../TripleStringStringOrNullTypeComputer.java | 42 -- .../impl/TypeCompatibilityChecker.java | 81 --- .../om/typecomputer/impl/TypeComputeUtils.java | 222 +++++++++ .../om/typecomputer/impl/TypeComputerUtils.java | 75 --- .../UnaryBinaryInt64OrNullTypeComputer.java | 67 --- .../impl/UnaryBinaryInt64TypeComputer.java | 50 ++ .../UnaryBooleanOrNullFunctionTypeComputer.java | 60 --- .../impl/UnaryMinusTypeComputer.java | 66 +++ .../UnaryStringInt64OrNullTypeComputer.java | 81 --- .../impl/UnaryStringInt64TypeComputer.java | 50 ++ .../impl/UnaryStringOrNullTypeComputer.java | 69 --- .../UnorderedListConstructorResultType.java | 32 +- .../apache/asterix/om/types/ARecordType.java | 2 +- .../org/apache/asterix/om/types/ATypeTag.java | 6 +- .../org/apache/asterix/om/types/AUnionType.java | 106 +++- .../apache/asterix/om/types/BuiltinType.java | 27 + .../org/apache/asterix/om/types/TypeHelper.java | 63 +-- .../apache/asterix/om/types/TypeTagUtil.java | 2 + .../asterix/om/util/NonTaggedFormatUtil.java | 58 ++- .../apache/asterix/om/visitors/IOMVisitor.java | 3 + .../om/visitors/OMPrintToStringVisitor.java | 10 +- asterixdb/asterix-runtime/pom.xml | 5 + ...bstractSerializableAvgAggregateFunction.java | 4 +- ...tractSerializableCountAggregateFunction.java | 4 +- ...bstractSerializableSumAggregateFunction.java | 4 +- .../std/SerializableAvgAggregateFunction.java | 2 +- .../SerializableGlobalAvgAggregateFunction.java | 2 +- ...lizableIntermediateAvgAggregateFunction.java | 2 +- .../SerializableLocalAvgAggregateFunction.java | 2 +- .../std/SerializableSumAggregateFunction.java | 2 +- .../std/AbstractAvgAggregateFunction.java | 2 +- .../std/AbstractCountAggregateFunction.java | 2 +- .../std/AbstractMinMaxAggregateFunction.java | 8 +- .../std/AbstractSumAggregateFunction.java | 5 +- .../aggregates/std/AvgAggregateFunction.java | 2 +- .../std/GlobalAvgAggregateFunction.java | 2 +- .../std/IntermediateAvgAggregateFunction.java | 2 +- .../std/LocalAvgAggregateFunction.java | 2 +- .../aggregates/std/MinMaxAggregateFunction.java | 2 +- .../accessors/CircleCenterAccessor.java | 7 - .../accessors/CircleRadiusAccessor.java | 7 - .../accessors/LineRectanglePolygonAccessor.java | 7 - .../accessors/PointXCoordinateAccessor.java | 7 - .../accessors/PointYCoordinateAccessor.java | 7 - .../accessors/TemporalDayAccessor.java | 8 - .../accessors/TemporalHourAccessor.java | 8 - .../accessors/TemporalIntervalEndAccessor.java | 10 +- .../TemporalIntervalEndDateAccessor.java | 11 +- .../TemporalIntervalEndDatetimeAccessor.java | 10 +- .../TemporalIntervalEndTimeAccessor.java | 10 +- .../TemporalIntervalStartAccessor.java | 10 +- .../TemporalIntervalStartDateAccessor.java | 10 +- .../TemporalIntervalStartDatetimeAccessor.java | 10 +- .../TemporalIntervalStartTimeAccessor.java | 10 +- .../accessors/TemporalMillisecondAccessor.java | 8 - .../accessors/TemporalMinuteAccessor.java | 8 - .../accessors/TemporalMonthAccessor.java | 8 - .../accessors/TemporalSecondAccessor.java | 8 - .../accessors/TemporalYearAccessor.java | 8 - .../common/AbstractTypeCheckEvaluator.java | 62 +++ .../ClosedRecordConstructorEvalFactory.java | 9 +- .../evaluators/common/CreateMBREvalFactory.java | 118 ++--- .../common/EditDistanceCheckEvaluator.java | 22 +- .../common/EditDistanceEvaluator.java | 32 +- .../common/SimilarityJaccardCheckEvaluator.java | 37 +- .../common/SimilarityJaccardEvaluator.java | 46 +- .../AbstractComparisonEvaluator.java | 125 ++--- .../comparisons/ComparisonEvalFactory.java | 171 ------- .../comparisons/EqualsDescriptor.java | 70 +++ .../comparisons/GreaterThanDescriptor.java | 70 +++ .../GreaterThanOrEqualsDescriptor.java | 70 +++ .../comparisons/LessThanDescriptor.java | 68 +++ .../comparisons/LessThanOrEqualsDescriptor.java | 70 +++ .../comparisons/NotEqualsDescriptor.java | 70 +++ .../ABinaryHexStringConstructorDescriptor.java | 14 +- .../ABooleanConstructorDescriptor.java | 7 - .../ACircleConstructorDescriptor.java | 6 - .../ADateConstructorDescriptor.java | 13 +- .../ADateTimeConstructorDescriptor.java | 6 - .../ADayTimeDurationConstructorDescriptor.java | 6 - .../ADoubleConstructorDescriptor.java | 6 - .../ADurationConstructorDescriptor.java | 6 - .../AFloatConstructorDescriptor.java | 6 - .../AInt16ConstructorDescriptor.java | 6 - .../AInt32ConstructorDescriptor.java | 6 - .../AInt64ConstructorDescriptor.java | 6 - .../AInt8ConstructorDescriptor.java | 6 - .../AIntervalConstructorDescriptor.java | 11 - ...ervalStartFromDateConstructorDescriptor.java | 11 - ...lStartFromDateTimeConstructorDescriptor.java | 11 - ...ervalStartFromTimeConstructorDescriptor.java | 11 - .../ALineConstructorDescriptor.java | 6 - .../ANullConstructorDescriptor.java | 8 +- .../APoint3DConstructorDescriptor.java | 6 - .../APointConstructorDescriptor.java | 6 - .../APolygonConstructorDescriptor.java | 9 - .../ARectangleConstructorDescriptor.java | 6 - .../AStringConstructorDescriptor.java | 14 +- .../ATimeConstructorDescriptor.java | 7 - .../AUUIDFromStringConstructorDescriptor.java | 6 - ...AYearMonthDurationConstructorDescriptor.java | 6 - .../OpenRecordConstructorDescriptor.java | 18 +- .../functions/AbstractBinaryStringBoolEval.java | 23 +- .../AbstractNumericArithmeticEval.java | 34 +- .../functions/AbstractQuadStringStringEval.java | 46 +- .../functions/AbstractTripleStringBoolEval.java | 43 +- .../AbstractTripleStringStringEval.java | 40 +- .../evaluators/functions/AndDescriptor.java | 37 +- .../AnyCollectionMemberDescriptor.java | 17 +- .../functions/CheckUnknownDescriptor.java | 86 ++++ .../functions/CreatePointDescriptor.java | 19 +- .../functions/DeepEqualityDescriptor.java | 2 +- .../functions/EditDistanceListIsFilterable.java | 158 ------ .../EditDistanceListIsFilterableDescriptor.java | 156 ++++++ .../EditDistanceStringIsFilterable.java | 171 ------- ...ditDistanceStringIsFilterableDescriptor.java | 66 +++ ...EditDistanceStringIsFilterableEvaluator.java | 132 +++++ .../functions/EmbedTypeDescriptor.java | 77 --- .../functions/FlowRecordDescriptor.java | 91 ---- .../evaluators/functions/FuzzyEqDescriptor.java | 49 -- .../evaluators/functions/GetItemDescriptor.java | 17 +- .../functions/IsMissingDescriptor.java | 67 +++ .../evaluators/functions/IsNullDescriptor.java | 34 +- .../functions/IsUnknownDescriptor.java | 68 +++ .../evaluators/functions/LenDescriptor.java | 15 - .../evaluators/functions/NotDescriptor.java | 6 - .../evaluators/functions/NotNullDescriptor.java | 85 ---- .../functions/NumericAbsDescriptor.java | 7 +- .../functions/NumericFloorDescriptor.java | 7 +- .../functions/NumericModuloDescriptor.java | 20 +- .../functions/NumericRoundDescriptor.java | 7 +- .../NumericRoundHalfToEven2Descriptor.java | 29 +- .../NumericRoundHalfToEvenDescriptor.java | 9 +- .../functions/NumericSubtractDescriptor.java | 8 - .../functions/NumericUnaryMinusDescriptor.java | 7 +- .../evaluators/functions/OrDescriptor.java | 30 +- .../functions/PrefixLenJaccardDescriptor.java | 14 +- .../evaluators/functions/RegExpDescriptor.java | 18 +- .../functions/SpatialAreaDescriptor.java | 11 - .../functions/SpatialCellDescriptor.java | 7 - .../functions/SpatialDistanceDescriptor.java | 12 +- .../functions/SpatialIntersectDescriptor.java | 20 +- .../functions/Substring2Descriptor.java | 20 +- .../functions/SubstringAfterDescriptor.java | 12 +- .../functions/SubstringBeforeDescriptor.java | 12 +- .../functions/SubstringDescriptor.java | 99 +--- .../binary/AbstractBinaryScalarEvaluator.java | 32 +- .../binary/AbstractFindBinaryEvaluator.java | 122 +++++ .../binary/AbstractSubBinaryEvaluator.java | 98 ++++ .../binary/BinaryConcatDescriptor.java | 22 +- .../binary/BinaryLengthDescriptor.java | 8 +- .../functions/binary/FindBinaryDescriptor.java | 102 +--- .../binary/FindBinaryFromDescriptor.java | 4 +- .../functions/binary/ParseBinaryDescriptor.java | 17 +- .../functions/binary/PrintBinaryDescriptor.java | 17 +- .../binary/SubBinaryFromDescriptor.java | 3 +- .../binary/SubBinaryFromToDescriptor.java | 76 +-- .../records/FieldAccessByIndexEvalFactory.java | 17 +- .../records/FieldAccessByNameEvalFactory.java | 17 +- .../records/FieldAccessNestedEvalFactory.java | 76 ++- .../records/GetRecordFieldValueEvalFactory.java | 21 +- .../records/GetRecordFieldsEvalFactory.java | 19 - .../records/RecordAddFieldsDescriptor.java | 28 +- .../records/RecordMergeDescriptor.java | 30 +- .../records/RecordRemoveFieldsEvalFactory.java | 21 +- .../AbstractIntervalLogicFuncDescriptor.java | 16 +- .../AdjustDateTimeForTimeZoneDescriptor.java | 15 - .../AdjustTimeForTimeZoneDescriptor.java | 16 - .../CalendarDuartionFromDateDescriptor.java | 12 - .../CalendarDurationFromDateTimeDescriptor.java | 12 - .../temporal/DateFromDatetimeDescriptor.java | 34 +- .../DateFromUnixTimeInDaysDescriptor.java | 15 +- .../DatetimeFromDateAndTimeDescriptor.java | 39 +- .../DatetimeFromUnixTimeInMsDescriptor.java | 43 +- .../DatetimeFromUnixTimeInSecsDescriptor.java | 48 +- .../functions/temporal/DayOfWeekDescriptor.java | 77 ++- .../DayTimeDurationComparatorDescriptor.java | 32 +- ...DurationGreaterThanComparatorDescriptor.java | 39 ++ ...imeDurationLessThanComparatorDescriptor.java | 39 ++ .../temporal/DurationEqualDescriptor.java | 12 - .../DurationFromIntervalDescriptor.java | 11 +- .../DurationFromMillisecondsDescriptor.java | 44 +- .../temporal/DurationFromMonthsDescriptor.java | 11 - .../temporal/GetDayTimeDurationDescriptor.java | 11 - .../GetOverlappingIntervalDescriptor.java | 5 +- .../GetYearMonthDurationDescriptor.java | 11 - .../temporal/IntervalBinDescriptor.java | 51 +- .../temporal/IntervalEndsDecriptor.java | 51 -- .../temporal/IntervalEndsDescriptor.java | 50 ++ ...llisecondsFromDayTimeDurationDescriptor.java | 11 - .../MonthsFromYearMonthDurationDescriptor.java | 11 - .../temporal/OverlapBinsDescriptor.java | 50 +- .../functions/temporal/ParseDateDescriptor.java | 11 - .../temporal/ParseDateTimeDescriptor.java | 11 - .../functions/temporal/ParseTimeDescriptor.java | 11 - .../functions/temporal/PrintDateDescriptor.java | 15 - .../temporal/PrintDateTimeDescriptor.java | 16 - .../functions/temporal/PrintTimeDescriptor.java | 15 - .../temporal/TimeFromDatetimeDescriptor.java | 31 +- .../TimeFromUnixTimeInMsDescriptor.java | 15 +- .../YearMonthDurationComparatorDecriptor.java | 155 ------ .../YearMonthDurationComparatorDescriptor.java | 125 +++++ ...DurationGreaterThanComparatorDescriptor.java | 39 ++ ...nthDurationLessThanComparatorDescriptor.java | 39 ++ .../evaluators/staticcodegen/CodeGenUtil.java | 91 +++- .../EvaluatorMissingCheckVisitor.java | 228 +++++++++ .../EvaluatorNullCheckVisitor.java | 105 ++++ .../staticcodegen/EvaluatorVisitor.java | 125 ----- .../staticcodegen/GatherInnerClassVisitor.java | 3 +- .../staticcodegen/RenameClassVisitor.java | 4 +- .../evaluators/staticcodegen/TypeCheckUtil.java | 41 -- .../evaluators/staticcodegen/TypeChecker.java | 77 +++ .../runtime/formats/NonTaggedDataFormat.java | 100 ++-- ...rixLSMPrimaryUpsertOperatorNodePushable.java | 22 +- .../AsterixLSMTreeUpsertOperatorDescriptor.java | 6 +- .../std/ScanCollectionDescriptor.java | 8 +- .../std/SubsetCollectionDescriptor.java | 11 +- .../asterix-server/src/main/licenses/NOTICE | 48 ++ .../asterix/tools/datagen/AdmDataGen.java | 2 +- .../api/AbstractCompilerFactoryBuilder.java | 24 +- .../api/HeuristicCompilerFactoryBuilder.java | 12 +- .../algebra/expressions/ConstantExpression.java | 22 +- .../expressions/IAlgebricksConstantValue.java | 2 + .../expressions/IMissableTypeComputer.java | 29 ++ .../expressions/INullableTypeComputer.java | 29 -- .../functions/AlgebricksBuiltinFunctions.java | 3 + .../logical/AbstractLogicalOperator.java | 2 +- .../operators/logical/GroupByOperator.java | 2 +- .../logical/LeftOuterJoinOperator.java | 4 +- .../logical/LeftOuterUnnestMapOperator.java | 2 +- .../logical/NestedTupleSourceOperator.java | 2 +- .../operators/logical/SelectOperator.java | 34 +- .../algebra/operators/logical/SinkOperator.java | 2 +- .../operators/logical/SubplanOperator.java | 2 +- .../physical/HybridHashJoinPOperator.java | 185 +++---- .../physical/InMemoryHashJoinPOperator.java | 4 +- .../operators/physical/NLJoinPOperator.java | 4 +- .../operators/physical/SubplanPOperator.java | 4 +- .../properties/TypePropagationPolicy.java | 55 ++- .../core/algebra/typing/ITypingContext.java | 4 +- .../typing/PropagatingTypeEnvironment.java | 6 +- .../algebra/util/OperatorPropertiesUtil.java | 4 +- .../core/jobgen/impl/JobGenContext.java | 16 +- .../base/AlgebricksOptimizationContext.java | 10 +- .../base/IOptimizationContextFactory.java | 4 +- .../data/impl/NoopMissingWriterFactory.java | 42 ++ .../data/impl/NoopNullWriterFactory.java | 47 -- .../examples/piglet/compiler/ConstantValue.java | 5 + .../rewriter/rules/PushSelectIntoJoinRule.java | 54 +- .../subplan/IntroduceGroupByForSubplanRule.java | 2 +- .../IntroduceLeftOuterJoinForSubplanRule.java | 2 +- .../rules/subplan/SubplanOutOfGroupRule.java | 2 +- .../operators/meta/SubplanRuntimeFactory.java | 18 +- .../std/StreamSelectRuntimeFactory.java | 48 +- .../tests/pushruntime/PushRuntimeTest.java | 6 +- .../value/BinaryComparatorConstant.java | 34 -- .../api/dataflow/value/IMissingWriter.java | 28 ++ .../dataflow/value/IMissingWriterFactory.java | 26 + .../hyracks/api/dataflow/value/INullWriter.java | 27 - .../api/dataflow/value/INullWriterFactory.java | 25 - .../join/GraceHashJoinOperatorDescriptor.java | 31 +- .../join/GraceHashJoinOperatorNodePushable.java | 16 +- .../join/HybridHashJoinOperatorDescriptor.java | 36 +- .../dataflow/std/join/InMemoryHashJoin.java | 24 +- .../InMemoryHashJoinOperatorDescriptor.java | 23 +- .../dataflow/std/join/NestedLoopJoin.java | 22 +- .../join/NestedLoopJoinOperatorDescriptor.java | 14 +- .../std/join/OptimizedHybridHashJoin.java | 14 +- ...timizedHybridHashJoinOperatorDescriptor.java | 118 ++--- .../TPCHCustomerOrderHashJoinTest.java | 28 +- .../TPCHCustomerOrderNestedLoopJoinTest.java | 12 +- .../tests/util/NoopMissingWriterFactory.java | 49 ++ .../tests/util/NoopNullWriterFactory.java | 49 -- .../dataflow/BTreeSearchOperatorDescriptor.java | 10 +- .../storage/am/btree/test/FramewriterTest.java | 2 +- .../AbstractIndexOperatorDescriptor.java | 10 +- .../AbstractTreeIndexOperatorDescriptor.java | 4 +- .../dataflow/IIndexOperatorDescriptor.java | 6 +- .../IndexSearchOperatorNodePushable.java | 33 +- ...dexInsertUpdateDeleteOperatorDescriptor.java | 6 +- ...tractLSMInvertedIndexOperatorDescriptor.java | 4 +- ...SMInvertedIndexSearchOperatorDescriptor.java | 4 +- .../dataflow/RTreeSearchOperatorDescriptor.java | 4 +- 552 files changed, 7817 insertions(+), 9447 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/535d86b5/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/jobgen/QueryLogicalExpressionJobGen.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/jobgen/QueryLogicalExpressionJobGen.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/jobgen/QueryLogicalExpressionJobGen.java index f4a926c..37f59dd 100644 --- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/jobgen/QueryLogicalExpressionJobGen.java +++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/jobgen/QueryLogicalExpressionJobGen.java @@ -26,7 +26,6 @@ import org.apache.asterix.formats.base.IDataFormat; import org.apache.asterix.om.functions.AsterixBuiltinFunctions; import org.apache.asterix.om.functions.IExternalFunctionInfo; import org.apache.asterix.om.functions.IFunctionDescriptor; -import org.apache.asterix.runtime.evaluators.comparisons.ComparisonEvalFactory; import org.apache.asterix.runtime.formats.FormatUtils; import org.apache.commons.lang3.mutable.Mutable; import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException; @@ -40,14 +39,11 @@ import org.apache.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvir import org.apache.hyracks.algebricks.core.algebra.expressions.StatefulFunctionCallExpression; import org.apache.hyracks.algebricks.core.algebra.expressions.UnnestingFunctionCallExpression; import org.apache.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression; -import org.apache.hyracks.algebricks.core.algebra.functions.AlgebricksBuiltinFunctions; -import org.apache.hyracks.algebricks.core.algebra.functions.AlgebricksBuiltinFunctions.ComparisonKind; -import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier; import org.apache.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema; import org.apache.hyracks.algebricks.core.jobgen.impl.JobGenContext; import org.apache.hyracks.algebricks.runtime.base.IAggregateEvaluatorFactory; -import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory; import org.apache.hyracks.algebricks.runtime.base.IRunningAggregateEvaluatorFactory; +import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory; import org.apache.hyracks.algebricks.runtime.base.ISerializedAggregateEvaluatorFactory; import org.apache.hyracks.algebricks.runtime.base.IUnnestingEvaluatorFactory; import org.apache.hyracks.algebricks.runtime.evaluators.ColumnAccessEvalFactory; @@ -62,7 +58,7 @@ public class QueryLogicalExpressionJobGen implements ILogicalExpressionJobGen { @Override public IAggregateEvaluatorFactory createAggregateFunctionFactory(AggregateFunctionCallExpression expr, IVariableTypeEnvironment env, IOperatorSchema[] inputSchemas, JobGenContext context) - throws AlgebricksException { + throws AlgebricksException { IScalarEvaluatorFactory[] args = codegenArguments(expr, env, inputSchemas, context); IFunctionDescriptor fd = getFunctionDescriptor(expr, env, context); switch (fd.getFunctionDescriptorTag()) { @@ -78,9 +74,9 @@ public class QueryLogicalExpressionJobGen implements ILogicalExpressionJobGen { } @Override - public IRunningAggregateEvaluatorFactory createRunningAggregateFunctionFactory( - StatefulFunctionCallExpression expr, IVariableTypeEnvironment env, IOperatorSchema[] inputSchemas, - JobGenContext context) throws AlgebricksException { + public IRunningAggregateEvaluatorFactory createRunningAggregateFunctionFactory(StatefulFunctionCallExpression expr, + IVariableTypeEnvironment env, IOperatorSchema[] inputSchemas, JobGenContext context) + throws AlgebricksException { IScalarEvaluatorFactory[] args = codegenArguments(expr, env, inputSchemas, context); return getFunctionDescriptor(expr, env, context).createRunningAggregateEvaluatorFactory(args); } @@ -88,7 +84,7 @@ public class QueryLogicalExpressionJobGen implements ILogicalExpressionJobGen { @Override public IUnnestingEvaluatorFactory createUnnestingFunctionFactory(UnnestingFunctionCallExpression expr, IVariableTypeEnvironment env, IOperatorSchema[] inputSchemas, JobGenContext context) - throws AlgebricksException { + throws AlgebricksException { IScalarEvaluatorFactory[] args = codegenArguments(expr, env, inputSchemas, context); return getFunctionDescriptor(expr, env, context).createUnnestingEvaluatorFactory(args); } @@ -133,14 +129,8 @@ public class QueryLogicalExpressionJobGen implements ILogicalExpressionJobGen { private IScalarEvaluatorFactory createScalarFunctionEvaluatorFactory(AbstractFunctionCallExpression expr, IVariableTypeEnvironment env, IOperatorSchema[] inputSchemas, JobGenContext context) - throws AlgebricksException { + throws AlgebricksException { IScalarEvaluatorFactory[] args = codegenArguments(expr, env, inputSchemas, context); - FunctionIdentifier fi = expr.getFunctionIdentifier(); - ComparisonKind ck = AlgebricksBuiltinFunctions.getComparisonType(fi); - if (ck != null) { - return new ComparisonEvalFactory(args[0], args[1], ck); - } - IFunctionDescriptor fd = null; if (!(expr.getFunctionInfo() instanceof IExternalFunctionInfo)) { IDataFormat format = FormatUtils.getDefaultFormat(); @@ -158,8 +148,9 @@ public class QueryLogicalExpressionJobGen implements ILogicalExpressionJobGen { return format.getConstantEvalFactory(expr.getValue()); } - private IScalarEvaluatorFactory[] codegenArguments(AbstractFunctionCallExpression expr, IVariableTypeEnvironment env, - IOperatorSchema[] inputSchemas, JobGenContext context) throws AlgebricksException { + private IScalarEvaluatorFactory[] codegenArguments(AbstractFunctionCallExpression expr, + IVariableTypeEnvironment env, IOperatorSchema[] inputSchemas, JobGenContext context) + throws AlgebricksException { List<Mutable<ILogicalExpression>> arguments = expr.getArguments(); int n = arguments.size(); IScalarEvaluatorFactory[] args = new IScalarEvaluatorFactory[n]; http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/535d86b5/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ByNameToByIndexFieldAccessRule.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ByNameToByIndexFieldAccessRule.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ByNameToByIndexFieldAccessRule.java index 20bfea5..b4f8ac3 100644 --- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ByNameToByIndexFieldAccessRule.java +++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ByNameToByIndexFieldAccessRule.java @@ -19,7 +19,6 @@ package org.apache.asterix.optimizer.rules; -import java.io.IOException; import java.util.ArrayList; import java.util.List; @@ -85,9 +84,9 @@ public class ByNameToByIndexFieldAccessRule implements IAlgebraicRewriteRule { ILogicalExpression a0 = fce.getArguments().get(0).getValue(); if (a0.getExpressionTag() == LogicalExpressionTag.FUNCTION_CALL) { LogicalVariable var1 = context.newVar(); - ArrayList<LogicalVariable> varArray = new ArrayList<LogicalVariable>(1); + ArrayList<LogicalVariable> varArray = new ArrayList<>(1); varArray.add(var1); - ArrayList<Mutable<ILogicalExpression>> exprArray = new ArrayList<Mutable<ILogicalExpression>>(1); + ArrayList<Mutable<ILogicalExpression>> exprArray = new ArrayList<>(1); exprArray.add(new MutableObject<ILogicalExpression>(a0)); AssignOperator assignVar = new AssignOperator(varArray, exprArray); fce.getArguments().get(0).setValue(new VariableReferenceExpression(var1)); @@ -100,44 +99,37 @@ public class ByNameToByIndexFieldAccessRule implements IAlgebraicRewriteRule { } IAType t = (IAType) env.getType(fce.getArguments().get(0).getValue()); - try { - switch (t.getTypeTag()) { - case ANY: { - return false || changed; + switch (t.getTypeTag()) { + case ANY: + return changed; + case RECORD: + ARecordType recType = (ARecordType) t; + ILogicalExpression fai = createFieldAccessByIndex(recType, fce); + if (fai == null) { + return changed; } - case RECORD: { - ARecordType recType = (ARecordType) t; - ILogicalExpression fai = createFieldAccessByIndex(recType, fce); - if (fai == null) { - return false || changed; - } - expressions.get(i).setValue(fai); - changed = true; - break; - } - case UNION: { - AUnionType unionT = (AUnionType) t; - if (unionT.isNullableType()) { - IAType t2 = unionT.getNullableType(); - if (t2.getTypeTag() == ATypeTag.RECORD) { - ARecordType recType = (ARecordType) t2; - ILogicalExpression fai = createFieldAccessByIndex(recType, fce); - if (fai == null) { - return false || changed; - } - expressions.get(i).setValue(fai); - changed = true; - break; - } - } + expressions.get(i).setValue(fai); + changed = true; + break; + case UNION: + AUnionType unionT = (AUnionType) t; + if (!unionT.isUnknownableType()) { throw new NotImplementedException("Union " + unionT); } - default: { + IAType t2 = unionT.getActualType(); + if (t2.getTypeTag() != ATypeTag.RECORD) { throw new AlgebricksException("Cannot call field-access on data of type " + t); } - } - } catch (IOException e) { - throw new AlgebricksException(e); + recType = (ARecordType) t2; + fai = createFieldAccessByIndex(recType, fce); + if (fai == null) { + return changed; + } + expressions.get(i).setValue(fai); + changed = true; + break; + default: + throw new AlgebricksException("Cannot call field-access on data of type " + t); } } assign.removeAnnotation(AsterixOperatorAnnotations.PUSHED_FIELD_ACCESS); @@ -145,8 +137,8 @@ public class ByNameToByIndexFieldAccessRule implements IAlgebraicRewriteRule { } @SuppressWarnings("unchecked") - private static ILogicalExpression createFieldAccessByIndex(ARecordType recType, AbstractFunctionCallExpression fce) - throws IOException { + private static ILogicalExpression createFieldAccessByIndex(ARecordType recType, + AbstractFunctionCallExpression fce) { String s = getStringSecondArgument(fce); if (s == null) { return null; http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/535d86b5/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/CheckFilterExpressionTypeRule.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/CheckFilterExpressionTypeRule.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/CheckFilterExpressionTypeRule.java index cda7619..1400950 100644 --- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/CheckFilterExpressionTypeRule.java +++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/CheckFilterExpressionTypeRule.java @@ -76,8 +76,8 @@ public class CheckFilterExpressionTypeRule implements IAlgebraicRewriteRule { */ private boolean isPossibleBoolean(IAType type) { while (NonTaggedFormatUtil.isOptional(type)) { - type = ((AUnionType) type).getNullableType(); - if (type.getTypeTag() == ATypeTag.BOOLEAN || type.getTypeTag() == ATypeTag.ANY) { + IAType actualType = ((AUnionType) type).getActualType(); + if (actualType.getTypeTag() == ATypeTag.BOOLEAN || actualType.getTypeTag() == ATypeTag.ANY) { return true; } } http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/535d86b5/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ConstantFoldingRule.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ConstantFoldingRule.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ConstantFoldingRule.java index fb48395..b29f110 100644 --- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ConstantFoldingRule.java +++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ConstantFoldingRule.java @@ -25,8 +25,7 @@ import java.util.List; import org.apache.asterix.common.config.GlobalConfig; import org.apache.asterix.dataflow.data.common.AqlExpressionTypeComputer; -import org.apache.asterix.dataflow.data.common.AqlNullableTypeComputer; -import org.apache.asterix.dataflow.data.nontagged.AqlNullWriterFactory; +import org.apache.asterix.dataflow.data.nontagged.AqlMissingWriterFactory; import org.apache.asterix.formats.nontagged.AqlADMPrinterFactoryProvider; import org.apache.asterix.formats.nontagged.AqlBinaryBooleanInspectorImpl; import org.apache.asterix.formats.nontagged.AqlBinaryComparatorFactoryProvider; @@ -40,7 +39,7 @@ import org.apache.asterix.om.base.AString; import org.apache.asterix.om.base.IAObject; import org.apache.asterix.om.constants.AsterixConstantValue; import org.apache.asterix.om.functions.AsterixBuiltinFunctions; -import org.apache.asterix.om.typecomputer.base.TypeComputerUtilities; +import org.apache.asterix.om.typecomputer.base.TypeCastUtils; import org.apache.asterix.om.types.ARecordType; import org.apache.asterix.om.types.ATypeTag; import org.apache.asterix.om.types.AbstractCollectionType; @@ -126,10 +125,9 @@ public class ConstantFoldingRule implements IAlgebraicRewriteRule { AqlSerializerDeserializerProvider.INSTANCE, AqlBinaryHashFunctionFactoryProvider.INSTANCE, AqlBinaryHashFunctionFamilyProvider.INSTANCE, AqlBinaryComparatorFactoryProvider.INSTANCE, AqlTypeTraitProvider.INSTANCE, AqlBinaryBooleanInspectorImpl.FACTORY, AqlBinaryIntegerInspector.FACTORY, - AqlADMPrinterFactoryProvider.INSTANCE, AqlNullWriterFactory.INSTANCE, null, + AqlADMPrinterFactoryProvider.INSTANCE, AqlMissingWriterFactory.INSTANCE, null, new LogicalExpressionJobGenToExpressionRuntimeProviderAdapter(QueryLogicalExpressionJobGen.INSTANCE), - AqlExpressionTypeComputer.INSTANCE, AqlNullableTypeComputer.INSTANCE, null, null, null, null, - GlobalConfig.DEFAULT_FRAME_SIZE, null); + AqlExpressionTypeComputer.INSTANCE, null, null, null, null, GlobalConfig.DEFAULT_FRAME_SIZE, null); private static final IOperatorSchema[] _emptySchemas = new IOperatorSchema[] {}; @@ -195,7 +193,7 @@ public class ConstantFoldingRule implements IAlgebraicRewriteRule { //Current List SerDe assumes a strongly typed list, so we do not constant fold the list constructors if they are not strongly typed if (expr.getFunctionIdentifier().equals(AsterixBuiltinFunctions.UNORDERED_LIST_CONSTRUCTOR) || expr.getFunctionIdentifier().equals(AsterixBuiltinFunctions.ORDERED_LIST_CONSTRUCTOR)) { - AbstractCollectionType listType = (AbstractCollectionType) TypeComputerUtilities.getRequiredType(expr); + AbstractCollectionType listType = (AbstractCollectionType) TypeCastUtils.getRequiredType(expr); if (listType != null && (listType.getItemType().getTypeTag() == ATypeTag.ANY || listType.getItemType() instanceof AbstractCollectionType)) { //case1: listType == null, could be a nested list inside a list<ANY> http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/535d86b5/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/FuzzyJoinRule.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/FuzzyJoinRule.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/FuzzyJoinRule.java index 427eda7..406d4ee 100644 --- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/FuzzyJoinRule.java +++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/FuzzyJoinRule.java @@ -31,8 +31,8 @@ import org.apache.asterix.lang.common.base.Clause; import org.apache.asterix.lang.common.struct.Identifier; import org.apache.asterix.metadata.declared.AqlMetadataProvider; import org.apache.asterix.om.functions.AsterixBuiltinFunctions; +import org.apache.asterix.om.typecomputer.impl.TypeComputeUtils; import org.apache.asterix.om.types.IAType; -import org.apache.asterix.om.types.TypeHelper; import org.apache.asterix.optimizer.base.FuzzyUtils; import org.apache.asterix.translator.AqlPlusExpressionToPlanTranslator; import org.apache.commons.lang3.mutable.Mutable; @@ -197,8 +197,8 @@ public class FuzzyJoinRule implements IAlgebraicRewriteRule { IAType leftType = (IAType) context.getOutputTypeEnvironment(leftInputOp).getVarType(leftInputVar); IAType rightType = (IAType) context.getOutputTypeEnvironment(rightInputOp).getVarType(rightInputVar); // left-hand side and right-hand side of "~=" has the same type - IAType left2 = TypeHelper.getNonOptionalType(leftType); - IAType right2 = TypeHelper.getNonOptionalType(rightType); + IAType left2 = TypeComputeUtils.getActualType(leftType); + IAType right2 = TypeComputeUtils.getActualType(rightType); if (!left2.deepEqual(right2)) { return false; } http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/535d86b5/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/InlineUnnestFunctionRule.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/InlineUnnestFunctionRule.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/InlineUnnestFunctionRule.java index 1c27a6e..092c561 100644 --- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/InlineUnnestFunctionRule.java +++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/InlineUnnestFunctionRule.java @@ -22,10 +22,9 @@ package org.apache.asterix.optimizer.rules; import java.util.ArrayList; import java.util.List; +import org.apache.asterix.om.functions.AsterixBuiltinFunctions; import org.apache.commons.lang3.mutable.Mutable; import org.apache.commons.lang3.mutable.MutableObject; - -import org.apache.asterix.om.functions.AsterixBuiltinFunctions; import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException; import org.apache.hyracks.algebricks.common.utils.Pair; import org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression; @@ -49,7 +48,8 @@ import org.apache.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule; public class InlineUnnestFunctionRule implements IAlgebraicRewriteRule { @Override - public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException { + public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) + throws AlgebricksException { return false; } @@ -57,20 +57,23 @@ public class InlineUnnestFunctionRule implements IAlgebraicRewriteRule { public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException { AbstractLogicalOperator op1 = (AbstractLogicalOperator) opRef.getValue(); - if (context.checkIfInDontApplySet(this, op1)) + if (context.checkIfInDontApplySet(this, op1)) { return false; + } context.addToDontApplySet(this, op1); - if (op1.getOperatorTag() != LogicalOperatorTag.UNNEST) + if (op1.getOperatorTag() != LogicalOperatorTag.UNNEST) { return false; + } UnnestOperator unnestOperator = (UnnestOperator) op1; AbstractFunctionCallExpression expr = (AbstractFunctionCallExpression) unnestOperator.getExpressionRef() .getValue(); //we only inline for the scan-collection function - if (expr.getFunctionIdentifier() != AsterixBuiltinFunctions.SCAN_COLLECTION) + if (expr.getFunctionIdentifier() != AsterixBuiltinFunctions.SCAN_COLLECTION) { return false; + } // inline all variables from an unnesting function call - AbstractFunctionCallExpression funcExpr = (AbstractFunctionCallExpression) expr; + AbstractFunctionCallExpression funcExpr = expr; List<Mutable<ILogicalExpression>> args = funcExpr.getArguments(); for (int i = 0; i < args.size(); i++) { ILogicalExpression argExpr = args.get(i).getValue(); @@ -95,8 +98,8 @@ public class InlineUnnestFunctionRule implements IAlgebraicRewriteRule { AbstractFunctionCallExpression expr = (AbstractFunctionCallExpression) unnestOp.getExpressionRef().getValue(); List<Pair<AbstractFunctionCallExpression, Integer>> parentAndIndexList = new ArrayList<Pair<AbstractFunctionCallExpression, Integer>>(); getParentFunctionExpression(usedVar, expr, parentAndIndexList); - ILogicalExpression usedVarOrginExpr = findUsedVarOrigin(usedVar, unnestOp, (AbstractLogicalOperator) unnestOp - .getInputs().get(0).getValue()); + ILogicalExpression usedVarOrginExpr = findUsedVarOrigin(usedVar, unnestOp, + (AbstractLogicalOperator) unnestOp.getInputs().get(0).getValue()); if (usedVarOrginExpr != null) { for (Pair<AbstractFunctionCallExpression, Integer> parentAndIndex : parentAndIndexList) { //we only rewrite the top scan-collection function @@ -116,8 +119,9 @@ public class InlineUnnestFunctionRule implements IAlgebraicRewriteRule { ILogicalExpression argExpr = args.get(i).getValue(); if (argExpr.getExpressionTag() == LogicalExpressionTag.VARIABLE) { VariableReferenceExpression varExpr = (VariableReferenceExpression) argExpr; - if (varExpr.getVariableReference().equals(usedVar)) + if (varExpr.getVariableReference().equals(usedVar)) { parentAndIndexList.add(new Pair<AbstractFunctionCallExpression, Integer>(funcExpr, i)); + } } if (argExpr.getExpressionTag() == LogicalExpressionTag.FUNCTION_CALL) { getParentFunctionExpression(usedVar, argExpr, parentAndIndexList); http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/535d86b5/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceAutogenerateIDRule.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceAutogenerateIDRule.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceAutogenerateIDRule.java index a7327ff..fee1c96 100644 --- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceAutogenerateIDRule.java +++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceAutogenerateIDRule.java @@ -130,7 +130,7 @@ public class IntroduceAutogenerateIDRule implements IAlgebraicRewriteRule { List<Mutable<ILogicalExpression>> args = new ArrayList<>(); args.add(new MutableObject<ILogicalExpression>(mergedRec)); AbstractFunctionCallExpression notNullFn = new ScalarFunctionCallExpression( - FunctionUtil.getFunctionInfo(AsterixBuiltinFunctions.NOT_NULL), args); + FunctionUtil.getFunctionInfo(AsterixBuiltinFunctions.CHECK_UNKNOWN), args); return notNullFn; } http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/535d86b5/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceDynamicTypeCastForExternalFunctionRule.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceDynamicTypeCastForExternalFunctionRule.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceDynamicTypeCastForExternalFunctionRule.java index d3370a5..0fb5b0b 100644 --- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceDynamicTypeCastForExternalFunctionRule.java +++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceDynamicTypeCastForExternalFunctionRule.java @@ -121,25 +121,25 @@ public class IntroduceDynamicTypeCastForExternalFunctionRule implements IAlgebra IAType inputRecordType = (IAType) env.getVarType(recordVar.get(0)); /** the input record type can be an union type -- for the case when it comes from a subplan or left-outer join */ - boolean checkNull = false; + boolean checkUnknown = false; while (NonTaggedFormatUtil.isOptional(inputRecordType)) { /** while-loop for the case there is a nested multi-level union */ - inputRecordType = ((AUnionType) inputRecordType).getNullableType(); - checkNull = true; + inputRecordType = ((AUnionType) inputRecordType).getActualType(); + checkUnknown = true; } /** see whether the input record type needs to be casted */ boolean cast = !IntroduceDynamicTypeCastRule.compatible(requiredRecordType, inputRecordType); - if (checkNull) { + if (checkUnknown) { recordVar.set(0, IntroduceDynamicTypeCastRule.addWrapperFunction(requiredRecordType, recordVar.get(0), - assignOp1, context, AsterixBuiltinFunctions.NOT_NULL)); + assignOp1, context, AsterixBuiltinFunctions.CHECK_UNKNOWN)); } if (cast) { IntroduceDynamicTypeCastRule.addWrapperFunction(requiredRecordType, recordVar.get(0), assignOp1, context, AsterixBuiltinFunctions.CAST_RECORD); } - return cast || checkNull; + return cast || checkUnknown; } } http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/535d86b5/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceDynamicTypeCastRule.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceDynamicTypeCastRule.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceDynamicTypeCastRule.java index f8bbba8..a58da0d 100644 --- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceDynamicTypeCastRule.java +++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceDynamicTypeCastRule.java @@ -25,7 +25,7 @@ import java.util.List; import org.apache.asterix.lang.common.util.FunctionUtil; import org.apache.asterix.metadata.declared.AqlDataSource; import org.apache.asterix.om.functions.AsterixBuiltinFunctions; -import org.apache.asterix.om.typecomputer.base.TypeComputerUtilities; +import org.apache.asterix.om.typecomputer.base.TypeCastUtils; import org.apache.asterix.om.types.ARecordType; import org.apache.asterix.om.types.ATypeTag; import org.apache.asterix.om.types.AUnionType; @@ -160,24 +160,24 @@ public class IntroduceDynamicTypeCastRule implements IAlgebraicRewriteRule { IAType inputRecordType = (IAType) env.getVarType(recordVar); /** the input record type can be an union type -- for the case when it comes from a subplan or left-outer join */ - boolean checkNull = false; + boolean checkUnknown = false; while (NonTaggedFormatUtil.isOptional(inputRecordType)) { /** while-loop for the case there is a nested multi-level union */ - inputRecordType = ((AUnionType) inputRecordType).getNullableType(); - checkNull = true; + inputRecordType = ((AUnionType) inputRecordType).getActualType(); + checkUnknown = true; } /** see whether the input record type needs to be casted */ boolean cast = !compatible(requiredRecordType, inputRecordType); - if (checkNull) { + if (checkUnknown) { recordVar = addWrapperFunction(requiredRecordType, recordVar, op, context, - AsterixBuiltinFunctions.NOT_NULL); + AsterixBuiltinFunctions.CHECK_UNKNOWN); } if (cast) { addWrapperFunction(requiredRecordType, recordVar, op, context, AsterixBuiltinFunctions.CAST_RECORD); } - return cast || checkNull; + return cast || checkUnknown; } /** @@ -217,7 +217,7 @@ public class IntroduceDynamicTypeCastRule implements IAlgebraicRewriteRule { cast.getArguments() .add(new MutableObject<ILogicalExpression>(new VariableReferenceExpression(var))); /** enforce the required record type */ - TypeComputerUtilities.setRequiredAndInputTypes(cast, requiredRecordType, actualType); + TypeCastUtils.setRequiredAndInputTypes(cast, requiredRecordType, actualType); LogicalVariable newAssignVar = context.newVar(); AssignOperator newAssignOperator = new AssignOperator(newAssignVar, new MutableObject<ILogicalExpression>(cast)); @@ -274,7 +274,7 @@ public class IntroduceDynamicTypeCastRule implements IAlgebraicRewriteRule { } IAType reqTypeInside = reqTypes[i]; if (NonTaggedFormatUtil.isOptional(reqTypes[i])) { - reqTypeInside = ((AUnionType) reqTypes[i]).getNullableType(); + reqTypeInside = ((AUnionType) reqTypes[i]).getActualType(); } IAType inputTypeInside = inputTypes[i]; if (NonTaggedFormatUtil.isOptional(inputTypes[i])) { @@ -282,9 +282,9 @@ public class IntroduceDynamicTypeCastRule implements IAlgebraicRewriteRule { /** if the required type is not optional, the two types are incompatible */ return false; } - inputTypeInside = ((AUnionType) inputTypes[i]).getNullableType(); + inputTypeInside = ((AUnionType) inputTypes[i]).getActualType(); } - if (inputTypeInside.getTypeTag() != ATypeTag.NULL && !reqTypeInside.equals(inputTypeInside)) { + if (inputTypeInside.getTypeTag() != ATypeTag.MISSING && !reqTypeInside.equals(inputTypeInside)) { return false; } } http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/535d86b5/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceEnforcedListTypeRule.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceEnforcedListTypeRule.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceEnforcedListTypeRule.java index 8b2e003..3ff2a1b 100644 --- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceEnforcedListTypeRule.java +++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceEnforcedListTypeRule.java @@ -24,7 +24,7 @@ import java.util.List; import org.apache.commons.lang3.mutable.Mutable; -import org.apache.asterix.om.typecomputer.base.TypeComputerUtilities; +import org.apache.asterix.om.typecomputer.base.TypeCastUtils; import org.apache.asterix.om.types.IAType; import org.apache.asterix.optimizer.rules.typecast.StaticTypeCastUtil; import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException; @@ -89,7 +89,7 @@ public class IntroduceEnforcedListTypeRule implements IAlgebraicRewriteRule { AbstractFunctionCallExpression argFuncExpr = (AbstractFunctionCallExpression) expr; IAType exprType = (IAType) env.getType(argFuncExpr); if (StaticTypeCastUtil.rewriteListExpr(argFuncExpr, exprType, exprType, env)) { - TypeComputerUtilities.resetRequiredAndInputTypes(argFuncExpr); + TypeCastUtils.resetRequiredAndInputTypes(argFuncExpr); changed = true; } } http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/535d86b5/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceSecondaryIndexInsertDeleteRule.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceSecondaryIndexInsertDeleteRule.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceSecondaryIndexInsertDeleteRule.java index e3cca9b..0a7d196 100644 --- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceSecondaryIndexInsertDeleteRule.java +++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceSecondaryIndexInsertDeleteRule.java @@ -41,7 +41,7 @@ import org.apache.asterix.om.base.AOrderedList; import org.apache.asterix.om.base.AString; import org.apache.asterix.om.constants.AsterixConstantValue; import org.apache.asterix.om.functions.AsterixBuiltinFunctions; -import org.apache.asterix.om.typecomputer.base.TypeComputerUtilities; +import org.apache.asterix.om.typecomputer.base.TypeCastUtils; import org.apache.asterix.om.types.AOrderedListType; import org.apache.asterix.om.types.ARecordType; import org.apache.asterix.om.types.ATypeTag; @@ -240,7 +240,7 @@ public class IntroduceSecondaryIndexInsertDeleteRule implements IAlgebraicRewrit // The first argument is the record castFunc.getArguments() .add(new MutableObject<ILogicalExpression>(insertOp.getPayloadExpression().getValue())); - TypeComputerUtilities.setRequiredAndInputTypes(castFunc, enforcedType, insertRecType); + TypeCastUtils.setRequiredAndInputTypes(castFunc, enforcedType, insertRecType); // AssignOperator puts in the cast var the casted record AssignOperator castedRecordAssignOperator = new AssignOperator(castedRecVar, new MutableObject<ILogicalExpression>(castFunc)); @@ -467,7 +467,7 @@ public class IntroduceSecondaryIndexInsertDeleteRule implements IAlgebraicRewrit new MutableObject<ILogicalExpression>(new VariableReferenceExpression(secondaryKeyVar))); } if (isPointMBR && isBulkload) { - //for PointMBR optimization: see SecondaryRTreeOperationsHelper.buildLoadingJobSpec() and + //for PointMBR optimization: see SecondaryRTreeOperationsHelper.buildLoadingJobSpec() and //createFieldPermutationForBulkLoadOp(int) for more details. for (LogicalVariable secondaryKeyVar : keyVarList) { secondaryExpressions.add(new MutableObject<ILogicalExpression>( @@ -502,7 +502,7 @@ public class IntroduceSecondaryIndexInsertDeleteRule implements IAlgebraicRewrit new VariableReferenceExpression(secondaryKeyVar))); } if (isPointMBR && isBulkload) { - //for PointMBR optimization: see SecondaryRTreeOperationsHelper.buildLoadingJobSpec() and + //for PointMBR optimization: see SecondaryRTreeOperationsHelper.buildLoadingJobSpec() and //createFieldPermutationForBulkLoadOp(int) for more details. for (LogicalVariable secondaryKeyVar : originalKeyVarList) { prevSecondaryExpressions.add(new MutableObject<ILogicalExpression>( @@ -620,8 +620,8 @@ public class IntroduceSecondaryIndexInsertDeleteRule implements IAlgebraicRewrit // if a an enforced field already exists and the type is correct IAType enforcedFieldType = recordNameTypesMap.get(splits.get(splits.size() - 1)); if (enforcedFieldType != null && enforcedFieldType.getTypeTag() == ATypeTag.UNION - && ((AUnionType) enforcedFieldType).isNullableType()) { - enforcedFieldType = ((AUnionType) enforcedFieldType).getNullableType(); + && ((AUnionType) enforcedFieldType).isUnknownableType()) { + enforcedFieldType = ((AUnionType) enforcedFieldType).getActualType(); } if (enforcedFieldType != null && !ATypeHierarchy.canPromote(enforcedFieldType.getTypeTag(), index.getKeyFieldTypes().get(i).getTypeTag())) { @@ -731,7 +731,7 @@ public class IntroduceSecondaryIndexInsertDeleteRule implements IAlgebraicRewrit continue; } ScalarFunctionCallExpression isNullFuncExpr = new ScalarFunctionCallExpression( - FunctionUtil.getFunctionInfo(AsterixBuiltinFunctions.IS_NULL), + FunctionUtil.getFunctionInfo(AsterixBuiltinFunctions.IS_UNKOWN), new MutableObject<ILogicalExpression>(new VariableReferenceExpression(secondaryKeyVar))); ScalarFunctionCallExpression notFuncExpr = new ScalarFunctionCallExpression( FunctionUtil.getFunctionInfo(AsterixBuiltinFunctions.NOT), http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/535d86b5/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceStaticTypeCastForInsertRule.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceStaticTypeCastForInsertRule.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceStaticTypeCastForInsertRule.java index 223fab3..89280be 100644 --- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceStaticTypeCastForInsertRule.java +++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceStaticTypeCastForInsertRule.java @@ -23,7 +23,7 @@ import java.util.ArrayList; import java.util.List; import org.apache.asterix.metadata.declared.AqlDataSource; -import org.apache.asterix.om.typecomputer.base.TypeComputerUtilities; +import org.apache.asterix.om.typecomputer.base.TypeCastUtils; import org.apache.asterix.om.types.IAType; import org.apache.asterix.optimizer.rules.typecast.StaticTypeCastUtil; import org.apache.commons.lang3.mutable.Mutable; @@ -144,7 +144,7 @@ public class IntroduceStaticTypeCastForInsertRule implements IAlgebraicRewriteRu AbstractFunctionCallExpression funcExpr = (AbstractFunctionCallExpression) expr; // that expression has been rewritten, and it will not // fail but just return false - if (TypeComputerUtilities.getRequiredType(funcExpr) != null) { + if (TypeCastUtils.getRequiredType(funcExpr) != null) { context.computeAndSetTypeEnvironmentForOperator(assignOp); return false; } http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/535d86b5/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ReplaceSinkOpWithCommitOpRule.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ReplaceSinkOpWithCommitOpRule.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ReplaceSinkOpWithCommitOpRule.java index 0628681..8aefd1a 100644 --- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ReplaceSinkOpWithCommitOpRule.java +++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ReplaceSinkOpWithCommitOpRule.java @@ -102,17 +102,17 @@ public class ReplaceSinkOpWithCommitOpRule implements IAlgebraicRewriteRule { upsertVar = context.newVar(); AbstractFunctionCallExpression orFunc = new ScalarFunctionCallExpression( FunctionUtil.getFunctionInfo(AsterixBuiltinFunctions.OR)); - // is new value null? -> this means that the expected operation is delete - AbstractFunctionCallExpression isNewNullFunc = new ScalarFunctionCallExpression( - FunctionUtil.getFunctionInfo(AsterixBuiltinFunctions.IS_NULL)); - isNewNullFunc.getArguments().add(insertDeleteUpsertOperator.getPayloadExpression()); - AbstractFunctionCallExpression isPrevNullFunc = new ScalarFunctionCallExpression( - FunctionUtil.getFunctionInfo(AsterixBuiltinFunctions.IS_NULL)); + // is new value missing? -> this means that the expected operation is delete + AbstractFunctionCallExpression isNewMissingFunc = new ScalarFunctionCallExpression( + FunctionUtil.getFunctionInfo(AsterixBuiltinFunctions.IS_MISSING)); + isNewMissingFunc.getArguments().add(insertDeleteUpsertOperator.getPayloadExpression()); + AbstractFunctionCallExpression isPrevMissingFunc = new ScalarFunctionCallExpression( + FunctionUtil.getFunctionInfo(AsterixBuiltinFunctions.IS_MISSING)); // argument is the previous record - isPrevNullFunc.getArguments().add(new MutableObject<ILogicalExpression>( + isPrevMissingFunc.getArguments().add(new MutableObject<ILogicalExpression>( new VariableReferenceExpression(insertDeleteUpsertOperator.getPrevRecordVar()))); - orFunc.getArguments().add(new MutableObject<ILogicalExpression>(isPrevNullFunc)); - orFunc.getArguments().add(new MutableObject<ILogicalExpression>(isNewNullFunc)); + orFunc.getArguments().add(new MutableObject<ILogicalExpression>(isPrevMissingFunc)); + orFunc.getArguments().add(new MutableObject<ILogicalExpression>(isNewMissingFunc)); // AssignOperator puts in the cast var the casted record upsertFlagAssign = new AssignOperator(upsertVar, new MutableObject<ILogicalExpression>(orFunc)); http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/535d86b5/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/SetClosedRecordConstructorsRule.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/SetClosedRecordConstructorsRule.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/SetClosedRecordConstructorsRule.java index 1af312d..ee55aa1 100644 --- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/SetClosedRecordConstructorsRule.java +++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/SetClosedRecordConstructorsRule.java @@ -21,7 +21,7 @@ package org.apache.asterix.optimizer.rules; import org.apache.asterix.common.config.GlobalConfig; import org.apache.asterix.lang.common.util.FunctionUtil; import org.apache.asterix.om.functions.AsterixBuiltinFunctions; -import org.apache.asterix.om.typecomputer.base.TypeComputerUtilities; +import org.apache.asterix.om.typecomputer.base.TypeCastUtils; import org.apache.asterix.om.types.AOrderedListType; import org.apache.asterix.om.types.ARecordType; import org.apache.asterix.om.types.AUnionType; @@ -108,7 +108,7 @@ public class SetClosedRecordConstructorsRule implements IAlgebraicRewriteRule { boolean allClosed = true; boolean changed = false; if (expr.getFunctionIdentifier().equals(AsterixBuiltinFunctions.OPEN_RECORD_CONSTRUCTOR)) { - ARecordType reqType = (ARecordType) TypeComputerUtilities.getRequiredType(expr); + ARecordType reqType = (ARecordType) TypeCastUtils.getRequiredType(expr); if (reqType == null || !reqType.isOpen()) { int n = expr.getArguments().size(); if (n % 2 > 0) { @@ -142,7 +142,7 @@ public class SetClosedRecordConstructorsRule implements IAlgebraicRewriteRule { boolean rewrite = true; if (expr.getFunctionIdentifier().equals(AsterixBuiltinFunctions.ORDERED_LIST_CONSTRUCTOR) || (expr.getFunctionIdentifier().equals(AsterixBuiltinFunctions.UNORDERED_LIST_CONSTRUCTOR))) { - IAType reqType = TypeComputerUtilities.getRequiredType(expr); + IAType reqType = TypeCastUtils.getRequiredType(expr); if (reqType == null) { rewrite = false; } @@ -172,45 +172,21 @@ public class SetClosedRecordConstructorsRule implements IAlgebraicRewriteRule { throw new AlgebricksException( "Could not infer type for variable '" + expr.getVariableReference() + "'."); } - boolean dataIsClosed = isClosedRec((IAType) varType); + boolean dataIsClosed = isClosed((IAType) varType); return new ClosedDataInfo(false, dataIsClosed, expr); } private boolean hasClosedType(ILogicalExpression expr) throws AlgebricksException { IAType t = (IAType) context.getExpressionTypeComputer().getType(expr, context.getMetadataProvider(), env); - return isClosedRec(t); + return isClosed(t); } - private static boolean isClosedRec(IAType t) throws AlgebricksException { + private static boolean isClosed(IAType t) throws AlgebricksException { switch (t.getTypeTag()) { + case MISSING: case ANY: { return false; } - case CIRCLE: - case INT8: - case INT16: - case INT32: - case INT64: - case BINARY: - case BITARRAY: - case FLOAT: - case DOUBLE: - case STRING: - case LINE: - case NULL: - case BOOLEAN: - case DATETIME: - case DATE: - case TIME: - case UUID: - case DURATION: - case YEARMONTHDURATION: - case DAYTIMEDURATION: - case INTERVAL: - case POINT: - case POINT3D: - case POLYGON: - case RECTANGLE: case SHORTWITHOUTTYPEINFO: { return true; } @@ -219,24 +195,21 @@ public class SetClosedRecordConstructorsRule implements IAlgebraicRewriteRule { } case UNION: { AUnionType ut = (AUnionType) t; - for (IAType t2 : ut.getUnionList()) { - if (!isClosedRec(t2)) { - return false; - } + if (!isClosed(ut.getActualType())) { + return false; } return true; } case ORDEREDLIST: { AOrderedListType ol = (AOrderedListType) t; - return isClosedRec(ol.getItemType()); + return isClosed(ol.getItemType()); } case UNORDEREDLIST: { AUnorderedListType ul = (AUnorderedListType) t; - return isClosedRec(ul.getItemType()); - } - default: { - throw new IllegalStateException("Closed type analysis not implemented for type " + t); + return isClosed(ul.getItemType()); } + default: + return true; } } } http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/535d86b5/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AbstractIntroduceAccessMethodRule.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AbstractIntroduceAccessMethodRule.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AbstractIntroduceAccessMethodRule.java index 0152c12..d17b663 100644 --- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AbstractIntroduceAccessMethodRule.java +++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AbstractIntroduceAccessMethodRule.java @@ -443,7 +443,7 @@ public abstract class AbstractIntroduceAccessMethodRule implements IAlgebraicRew // Need to also verify the index is pending no op if (index.getKeyFieldNames().contains(fieldName) && index.getPendingOp() == IMetadataEntity.PENDING_NO_OP) { indexCandidates.add(index); - if (optFuncExpr.getFieldType(varIdx) == BuiltinType.ANULL + if (optFuncExpr.getFieldType(varIdx) == BuiltinType.AMISSING || optFuncExpr.getFieldType(varIdx) == BuiltinType.ANY) { optFuncExpr.setFieldType(varIdx, index.getKeyFieldTypes().get(index.getKeyFieldNames().indexOf(fieldName))); http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/535d86b5/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AccessMethodUtils.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AccessMethodUtils.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AccessMethodUtils.java index 2ce43c2..db2c627 100644 --- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AccessMethodUtils.java +++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AccessMethodUtils.java @@ -516,7 +516,7 @@ public class AccessMethodUtils { return primaryIndexUnnestOp; } - public static ScalarFunctionCallExpression findLOJIsNullFuncInGroupBy(GroupByOperator lojGroupbyOp) + public static ScalarFunctionCallExpression findLOJIsMissingFuncInGroupBy(GroupByOperator lojGroupbyOp) throws AlgebricksException { //find IS_NULL function of which argument has the nullPlaceholder variable in the nested plan of groupby. ALogicalPlanImpl subPlan = (ALogicalPlanImpl) lojGroupbyOp.getNestedPlans().get(0); @@ -536,7 +536,7 @@ public class AccessMethodUtils { if (notFuncExpr.getArguments().get(0).getValue() .getExpressionTag() == LogicalExpressionTag.FUNCTION_CALL) { if (((AbstractFunctionCallExpression) notFuncExpr.getArguments().get(0).getValue()) - .getFunctionIdentifier().equals(AlgebricksBuiltinFunctions.IS_NULL)) { + .getFunctionIdentifier().equals(AlgebricksBuiltinFunctions.IS_MISSING)) { isNullFuncExpr = (ScalarFunctionCallExpression) notFuncExpr.getArguments().get(0) .getValue(); if (isNullFuncExpr.getArguments().get(0).getValue() http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/535d86b5/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/IntroduceJoinAccessMethodRule.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/IntroduceJoinAccessMethodRule.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/IntroduceJoinAccessMethodRule.java index 8a9bb3f..e1cda09 100644 --- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/IntroduceJoinAccessMethodRule.java +++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/IntroduceJoinAccessMethodRule.java @@ -158,7 +158,7 @@ public class IntroduceJoinAccessMethodRule extends AbstractIntroduceAccessMethod if (isLeftOuterJoin && hasGroupBy) { analysisCtx.setLOJGroupbyOpRef(opRef); ScalarFunctionCallExpression isNullFuncExpr = AccessMethodUtils - .findLOJIsNullFuncInGroupBy((GroupByOperator) opRef.getValue()); + .findLOJIsMissingFuncInGroupBy((GroupByOperator) opRef.getValue()); analysisCtx.setLOJIsNullFuncInGroupBy(isNullFuncExpr); }
