Dmitry Lychagin has submitted this change and it was merged. Change subject: [ASTERIXDB-2203][IDX] Eliminate sort operator for secondary primary index ......................................................................
[ASTERIXDB-2203][IDX] Eliminate sort operator for secondary primary index - user model changes: no - storage format changes: no - interface changes: no details: This change is intended to remove the sort operator when creating a secondary primary index. The secondary primary index DDL statement uses the same code as the one for creating general secondary BTree indexes which has the sort operator since it needs to sort by SK. The sort operator is not needed for secondary primary index since there is no SK and PK is already sorted. Change-Id: I2287c355204355b7ce306388c85d58d69a1bb64a Reviewed-on: https://asterix-gerrit.ics.uci.edu/2240 Sonar-Qube: Jenkins <[email protected]> Integration-Tests: Jenkins <[email protected]> Tested-by: Jenkins <[email protected]> Contrib: Jenkins <[email protected]> Reviewed-by: Dmitry Lychagin <[email protected]> --- M asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryBTreeOperationsHelper.java 1 file changed, 39 insertions(+), 32 deletions(-) Approvals: Anon. E. Moose #1000171: Jenkins: Verified; No violations found; ; Verified Dmitry Lychagin: Looks good to me, approved diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryBTreeOperationsHelper.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryBTreeOperationsHelper.java index 41def96..cf6338a 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryBTreeOperationsHelper.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryBTreeOperationsHelper.java @@ -125,50 +125,57 @@ spec.setConnectorPolicyAssignmentPolicy(new ConnectorPolicyAssignmentPolicy()); return spec; } else { - // Create dummy key provider for feeding the primary index scan. - IOperatorDescriptor keyProviderOp = DatasetUtil.createDummyKeyProviderOp(spec, dataset, - metadataProvider); + // job spec: + // key provider -> primary idx -> (cast assign)? -> assign -> (select)? -> (sort)? -> bulk load -> sink IndexUtil.bindJobEventListener(spec, metadataProvider); - // Create primary index scan op. - IOperatorDescriptor primaryScanOp = DatasetUtil.createPrimaryIndexScanOp(spec, metadataProvider, dataset); + // dummy key provider ----> primary index scan + IOperatorDescriptor sourceOp = DatasetUtil.createDummyKeyProviderOp(spec, dataset, metadataProvider); + IOperatorDescriptor targetOp = DatasetUtil.createPrimaryIndexScanOp(spec, metadataProvider, dataset); + spec.connect(new OneToOneConnectorDescriptor(spec), sourceOp, 0, targetOp, 0); - // Assign op. - IOperatorDescriptor sourceOp = primaryScanOp; + sourceOp = targetOp; if (isOverridingKeyFieldTypes && !enforcedItemType.equals(itemType)) { - sourceOp = createCastOp(spec, dataset.getDatasetType(), index.isEnforced()); - spec.connect(new OneToOneConnectorDescriptor(spec), primaryScanOp, 0, sourceOp, 0); + // primary index scan ----> cast assign + targetOp = createCastOp(spec, dataset.getDatasetType(), index.isEnforced()); + spec.connect(new OneToOneConnectorDescriptor(spec), sourceOp, 0, targetOp, 0); + sourceOp = targetOp; } - AlgebricksMetaOperatorDescriptor asterixAssignOp = - createAssignOp(spec, index.getKeyFieldNames().size(), secondaryRecDesc); + // primary index OR cast assign ----> assign op + targetOp = createAssignOp(spec, index.getKeyFieldNames().size(), secondaryRecDesc); + spec.connect(new OneToOneConnectorDescriptor(spec), sourceOp, 0, targetOp, 0); - // If any of the secondary fields are nullable, then add a select op that filters nulls. - AlgebricksMetaOperatorDescriptor selectOp = null; + sourceOp = targetOp; if (anySecondaryKeyIsNullable || isOverridingKeyFieldTypes) { - selectOp = createFilterNullsSelectOp(spec, index.getKeyFieldNames().size(), secondaryRecDesc); + // if any of the secondary fields are nullable, then add a select op that filters nulls. + // assign op ----> select op + targetOp = createFilterNullsSelectOp(spec, index.getKeyFieldNames().size(), secondaryRecDesc); + spec.connect(new OneToOneConnectorDescriptor(spec), sourceOp, 0, targetOp, 0); + sourceOp = targetOp; } - // Sort by secondary keys. - ExternalSortOperatorDescriptor sortOp = createSortOp(spec, secondaryComparatorFactories, secondaryRecDesc); - // Create secondary BTree bulk load op. - TreeIndexBulkLoadOperatorDescriptor secondaryBulkLoadOp = createTreeIndexBulkLoadOp(spec, fieldPermutation, - dataflowHelperFactory, GlobalConfig.DEFAULT_TREE_FILL_FACTOR); + // no need to sort if the index is secondary primary index + if (!index.getKeyFieldNames().isEmpty()) { + // sort by secondary keys. + // assign op OR select op ----> sort op + targetOp = createSortOp(spec, secondaryComparatorFactories, secondaryRecDesc); + spec.connect(new OneToOneConnectorDescriptor(spec), sourceOp, 0, targetOp, 0); + sourceOp = targetOp; + } - AlgebricksMetaOperatorDescriptor metaOp = new AlgebricksMetaOperatorDescriptor(spec, 1, 0, + // assign op OR select op OR sort op ----> bulk load op + targetOp = createTreeIndexBulkLoadOp(spec, fieldPermutation, dataflowHelperFactory, + GlobalConfig.DEFAULT_TREE_FILL_FACTOR); + spec.connect(new OneToOneConnectorDescriptor(spec), sourceOp, 0, targetOp, 0); + + // bulk load op ----> sink op + sourceOp = targetOp; + targetOp = new AlgebricksMetaOperatorDescriptor(spec, 1, 0, new IPushRuntimeFactory[] { new SinkRuntimeFactory() }, new RecordDescriptor[] { secondaryRecDesc }); - // Connect the operators. - spec.connect(new OneToOneConnectorDescriptor(spec), keyProviderOp, 0, primaryScanOp, 0); - spec.connect(new OneToOneConnectorDescriptor(spec), sourceOp, 0, asterixAssignOp, 0); - if (anySecondaryKeyIsNullable || isOverridingKeyFieldTypes) { - spec.connect(new OneToOneConnectorDescriptor(spec), asterixAssignOp, 0, selectOp, 0); - spec.connect(new OneToOneConnectorDescriptor(spec), selectOp, 0, sortOp, 0); - } else { - spec.connect(new OneToOneConnectorDescriptor(spec), asterixAssignOp, 0, sortOp, 0); - } - spec.connect(new OneToOneConnectorDescriptor(spec), sortOp, 0, secondaryBulkLoadOp, 0); - spec.connect(new OneToOneConnectorDescriptor(spec), secondaryBulkLoadOp, 0, metaOp, 0); - spec.addRoot(metaOp); + spec.connect(new OneToOneConnectorDescriptor(spec), sourceOp, 0, targetOp, 0); + + spec.addRoot(targetOp); spec.setConnectorPolicyAssignmentPolicy(new ConnectorPolicyAssignmentPolicy()); return spec; } -- To view, visit https://asterix-gerrit.ics.uci.edu/2240 To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-MessageType: merged Gerrit-Change-Id: I2287c355204355b7ce306388c85d58d69a1bb64a Gerrit-PatchSet: 2 Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Owner: Ali Alsuliman <[email protected]> Gerrit-Reviewer: Anon. E. Moose #1000171 Gerrit-Reviewer: Dmitry Lychagin <[email protected]> Gerrit-Reviewer: Jenkins <[email protected]> Gerrit-Reviewer: Till Westmann <[email protected]>
