Repository: asterixdb Updated Branches: refs/heads/master ee30e8bde -> 4c5fb5c31
[ASTERIXDB-2203][IDX] Eliminate sort operator for secondary primary index - user model changes: no - storage format changes: no - interface changes: no details: This change is intended to remove the sort operator when creating a secondary primary index. The secondary primary index DDL statement uses the same code as the one for creating general secondary BTree indexes which has the sort operator since it needs to sort by SK. The sort operator is not needed for secondary primary index since there is no SK and PK is already sorted. Change-Id: I2287c355204355b7ce306388c85d58d69a1bb64a Reviewed-on: https://asterix-gerrit.ics.uci.edu/2240 Sonar-Qube: Jenkins <[email protected]> Integration-Tests: Jenkins <[email protected]> Tested-by: Jenkins <[email protected]> Contrib: Jenkins <[email protected]> Reviewed-by: Dmitry Lychagin <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/4c5fb5c3 Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/4c5fb5c3 Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/4c5fb5c3 Branch: refs/heads/master Commit: 4c5fb5c313b60c6b8b0fb0812cec8cc98c0289a0 Parents: ee30e8b Author: Ali Alsuliman <[email protected]> Authored: Tue Dec 19 14:16:31 2017 -0800 Committer: Dmitry Lychagin <[email protected]> Committed: Wed Dec 20 14:30:25 2017 -0800 ---------------------------------------------------------------------- .../utils/SecondaryBTreeOperationsHelper.java | 71 +++++++++++--------- 1 file changed, 39 insertions(+), 32 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4c5fb5c3/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryBTreeOperationsHelper.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryBTreeOperationsHelper.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryBTreeOperationsHelper.java index 41def96..cf6338a 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryBTreeOperationsHelper.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryBTreeOperationsHelper.java @@ -125,50 +125,57 @@ public class SecondaryBTreeOperationsHelper extends SecondaryTreeIndexOperations spec.setConnectorPolicyAssignmentPolicy(new ConnectorPolicyAssignmentPolicy()); return spec; } else { - // Create dummy key provider for feeding the primary index scan. - IOperatorDescriptor keyProviderOp = DatasetUtil.createDummyKeyProviderOp(spec, dataset, - metadataProvider); + // job spec: + // key provider -> primary idx -> (cast assign)? -> assign -> (select)? -> (sort)? -> bulk load -> sink IndexUtil.bindJobEventListener(spec, metadataProvider); - // Create primary index scan op. - IOperatorDescriptor primaryScanOp = DatasetUtil.createPrimaryIndexScanOp(spec, metadataProvider, dataset); + // dummy key provider ----> primary index scan + IOperatorDescriptor sourceOp = DatasetUtil.createDummyKeyProviderOp(spec, dataset, metadataProvider); + IOperatorDescriptor targetOp = DatasetUtil.createPrimaryIndexScanOp(spec, metadataProvider, dataset); + spec.connect(new OneToOneConnectorDescriptor(spec), sourceOp, 0, targetOp, 0); - // Assign op. - IOperatorDescriptor sourceOp = primaryScanOp; + sourceOp = targetOp; if (isOverridingKeyFieldTypes && !enforcedItemType.equals(itemType)) { - sourceOp = createCastOp(spec, dataset.getDatasetType(), index.isEnforced()); - spec.connect(new OneToOneConnectorDescriptor(spec), primaryScanOp, 0, sourceOp, 0); + // primary index scan ----> cast assign + targetOp = createCastOp(spec, dataset.getDatasetType(), index.isEnforced()); + spec.connect(new OneToOneConnectorDescriptor(spec), sourceOp, 0, targetOp, 0); + sourceOp = targetOp; } - AlgebricksMetaOperatorDescriptor asterixAssignOp = - createAssignOp(spec, index.getKeyFieldNames().size(), secondaryRecDesc); + // primary index OR cast assign ----> assign op + targetOp = createAssignOp(spec, index.getKeyFieldNames().size(), secondaryRecDesc); + spec.connect(new OneToOneConnectorDescriptor(spec), sourceOp, 0, targetOp, 0); - // If any of the secondary fields are nullable, then add a select op that filters nulls. - AlgebricksMetaOperatorDescriptor selectOp = null; + sourceOp = targetOp; if (anySecondaryKeyIsNullable || isOverridingKeyFieldTypes) { - selectOp = createFilterNullsSelectOp(spec, index.getKeyFieldNames().size(), secondaryRecDesc); + // if any of the secondary fields are nullable, then add a select op that filters nulls. + // assign op ----> select op + targetOp = createFilterNullsSelectOp(spec, index.getKeyFieldNames().size(), secondaryRecDesc); + spec.connect(new OneToOneConnectorDescriptor(spec), sourceOp, 0, targetOp, 0); + sourceOp = targetOp; } - // Sort by secondary keys. - ExternalSortOperatorDescriptor sortOp = createSortOp(spec, secondaryComparatorFactories, secondaryRecDesc); - // Create secondary BTree bulk load op. - TreeIndexBulkLoadOperatorDescriptor secondaryBulkLoadOp = createTreeIndexBulkLoadOp(spec, fieldPermutation, - dataflowHelperFactory, GlobalConfig.DEFAULT_TREE_FILL_FACTOR); + // no need to sort if the index is secondary primary index + if (!index.getKeyFieldNames().isEmpty()) { + // sort by secondary keys. + // assign op OR select op ----> sort op + targetOp = createSortOp(spec, secondaryComparatorFactories, secondaryRecDesc); + spec.connect(new OneToOneConnectorDescriptor(spec), sourceOp, 0, targetOp, 0); + sourceOp = targetOp; + } - AlgebricksMetaOperatorDescriptor metaOp = new AlgebricksMetaOperatorDescriptor(spec, 1, 0, + // assign op OR select op OR sort op ----> bulk load op + targetOp = createTreeIndexBulkLoadOp(spec, fieldPermutation, dataflowHelperFactory, + GlobalConfig.DEFAULT_TREE_FILL_FACTOR); + spec.connect(new OneToOneConnectorDescriptor(spec), sourceOp, 0, targetOp, 0); + + // bulk load op ----> sink op + sourceOp = targetOp; + targetOp = new AlgebricksMetaOperatorDescriptor(spec, 1, 0, new IPushRuntimeFactory[] { new SinkRuntimeFactory() }, new RecordDescriptor[] { secondaryRecDesc }); - // Connect the operators. - spec.connect(new OneToOneConnectorDescriptor(spec), keyProviderOp, 0, primaryScanOp, 0); - spec.connect(new OneToOneConnectorDescriptor(spec), sourceOp, 0, asterixAssignOp, 0); - if (anySecondaryKeyIsNullable || isOverridingKeyFieldTypes) { - spec.connect(new OneToOneConnectorDescriptor(spec), asterixAssignOp, 0, selectOp, 0); - spec.connect(new OneToOneConnectorDescriptor(spec), selectOp, 0, sortOp, 0); - } else { - spec.connect(new OneToOneConnectorDescriptor(spec), asterixAssignOp, 0, sortOp, 0); - } - spec.connect(new OneToOneConnectorDescriptor(spec), sortOp, 0, secondaryBulkLoadOp, 0); - spec.connect(new OneToOneConnectorDescriptor(spec), secondaryBulkLoadOp, 0, metaOp, 0); - spec.addRoot(metaOp); + spec.connect(new OneToOneConnectorDescriptor(spec), sourceOp, 0, targetOp, 0); + + spec.addRoot(targetOp); spec.setConnectorPolicyAssignmentPolicy(new ConnectorPolicyAssignmentPolicy()); return spec; }
