>From Ali Alsuliman <[email protected]>:
Ali Alsuliman has uploaded this change for review. (
https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/19877 )
Change subject: WIP: heterogenous index cast
......................................................................
WIP: heterogenous index cast
Change-Id: I99f46564bc0879f2a82b0cd7dfbbd95132be7512
---
M
asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryIndexOperationsHelper.java
M
asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/BTreeAccessMethod.java
M
asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceSecondaryIndexInsertDeleteRule.java
3 files changed, 81 insertions(+), 9 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb
refs/changes/77/19877/1
diff --git
a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceSecondaryIndexInsertDeleteRule.java
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceSecondaryIndexInsertDeleteRule.java
index df8d4cd..33c5baa 100644
---
a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceSecondaryIndexInsertDeleteRule.java
+++
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceSecondaryIndexInsertDeleteRule.java
@@ -860,7 +860,7 @@
AbstractFunctionCallExpression theFieldAccessFunc;
LogicalVariable fieldVar = context.newVar();
- if (fieldType == null &&
!ATypeTag.ANY.equals(skType.getTypeTag())) {
+ if (fieldType == null) {
// Open field. must prevent inlining to maintain the cast
before the primaryOp and
// make handling of records with incorrect value type for
this field easier and cleaner
context.addNotToBeInlinedVar(fieldVar);
@@ -871,6 +871,13 @@
theFieldAccessFunc = createCastExpression(index, skType,
fieldAccessFunc, sourceLoc,
indexFieldId.funId, indexFieldId.extraArg);
} else {
+ // sanity check. (heterogeneous) index keys with type ANY
should not be on typed fields
+ ATypeTag skTag = skType.getTypeTag();
+ if (ATypeTag.ANY.equals(skTag)) {
+ throw new
CompilationException(ErrorCode.COMPILATION_ILLEGAL_STATE, sourceLoc,
+ "type mismatch for field " +
indexFieldId.fieldName + ". secondary key type " + skTag
+ + ", field type " +
fieldType.getTypeTag());
+ }
// Get the desired field position
int pos = indexFieldId.fieldName.size() > 1 ? -1
:
sourceType.getFieldIndex(indexFieldId.fieldName.get(0));
@@ -903,12 +910,13 @@
private static IndexFieldId createIndexFieldId(Index index, List<String>
skName, IAType skType, Integer skSrc,
ARecordType sourceType, SourceLocation srcLoc) throws
AlgebricksException {
IAType fieldType = sourceType.getSubFieldType(skName);
- FunctionIdentifier skFun = null;
- IAObject fmtArg = null;
Pair<FunctionIdentifier, IAObject> castExpr;
if (ATypeTag.ANY.equals(skType.getTypeTag())) {
- return new IndexFieldId(skSrc, skName, skType.getTypeTag(), skFun,
fmtArg);
+ // heterogeneous index needs to make sure the keys are in the open
format (for records & lists values)
+ return new IndexFieldId(skSrc, skName, skType.getTypeTag(),
BuiltinFunctions.CAST_TYPE, null);
}
+ FunctionIdentifier skFun = null;
+ IAObject fmtArg = null;
if (fieldType == null) {
// open field
castExpr = getCastExpression(index, skType, srcLoc);
diff --git
a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/BTreeAccessMethod.java
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/BTreeAccessMethod.java
index b4c9484..d82a8be 100644
---
a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/BTreeAccessMethod.java
+++
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/BTreeAccessMethod.java
@@ -48,8 +48,10 @@
import org.apache.asterix.om.base.ADouble;
import org.apache.asterix.om.base.IAObject;
import org.apache.asterix.om.functions.BuiltinFunctions;
+import org.apache.asterix.om.typecomputer.base.TypeCastUtils;
import org.apache.asterix.om.typecomputer.impl.TypeComputeUtils;
import org.apache.asterix.om.types.ARecordType;
+import org.apache.asterix.om.types.ATypeTag;
import org.apache.asterix.om.types.BuiltinType;
import org.apache.asterix.om.types.IAType;
import org.apache.asterix.om.types.hierachy.ATypeHierarchy;
@@ -86,6 +88,7 @@
import
org.apache.hyracks.algebricks.core.algebra.operators.logical.SelectOperator;
import
org.apache.hyracks.algebricks.core.algebra.operators.logical.UnnestMapOperator;
import
org.apache.hyracks.algebricks.core.algebra.util.OperatorManipulationUtil;
+import org.apache.hyracks.api.exceptions.SourceLocation;
import org.apache.hyracks.util.LogRedactionUtil;
/**
@@ -348,6 +351,7 @@
// we made sure indexSubTree has datasource scan
AbstractDataSourceOperator dataSourceOp =
(AbstractDataSourceOperator)
indexSubTree.getDataSourceRef().getValue();
+ SourceLocation dataSrcLoc = dataSourceOp.getSourceLocation();
List<Pair<Integer, Integer>> exprAndVarList =
analysisCtx.getIndexExprsFromIndexExprsAndVars(chosenIndex);
int numSecondaryKeys = analysisCtx.getNumberOfMatchedKeys(chosenIndex);
@@ -643,7 +647,7 @@
if (!assignKeyVarList.isEmpty()) {
// Assign operator that sets the constant secondary-index
search-key fields if necessary.
AssignOperator assignSearchKeys = new
AssignOperator(assignKeyVarList, assignKeyExprList);
-
assignSearchKeys.setSourceLocation(dataSourceOp.getSourceLocation());
+ assignSearchKeys.setSourceLocation(dataSrcLoc);
if (probeSubTree == null) {
// We are optimizing a selection query.
// Input to this assign is the EmptyTupleSource (which the
dataSourceScan also must have had as input).
@@ -674,6 +678,11 @@
inputOp = probeSubTree.getRoot();
}
+ // if a key is of type ANY, we need to cast the search value to ANY if
it is record/list for proper comparison
+ if (chosenIndexKeyFieldTypes.stream().anyMatch(t -> t.getTypeTag() ==
ATypeTag.ANY)) {
+ inputOp = addCastAssignOp(context, chosenIndexKeyFieldTypes,
jobGenParams, inputOp, dataSrcLoc);
+ }
+
// Creates an unnest-map for the secondary index search.
// The result: SK, PK, [Optional - the result of an instantTrylock on
PK]
ILogicalOperator secondaryIndexUnnestOp =
AccessMethodUtils.createSecondaryIndexUnnestMap(dataset, recordType,
@@ -763,7 +772,7 @@
IFunctionInfo primaryIndexSearch =
FunctionUtil.getFunctionInfo(BuiltinFunctions.INDEX_SEARCH);
UnnestingFunctionCallExpression primaryIndexSearchFunc =
new
UnnestingFunctionCallExpression(primaryIndexSearch, primaryIndexFuncArgs);
-
primaryIndexSearchFunc.setSourceLocation(dataSourceOp.getSourceLocation());
+ primaryIndexSearchFunc.setSourceLocation(dataSrcLoc);
primaryIndexSearchFunc.setReturnsUniqueValues(true);
if (!leftOuterUnnestMapRequired) {
unnestMapOp = new UnnestMapOperator(scanVariables, new
MutableObject<>(primaryIndexSearchFunc),
@@ -785,7 +794,7 @@
}
}
unnestMapOp.setExecutionMode(ExecutionMode.PARTITIONED);
- unnestMapOp.setSourceLocation(dataSourceOp.getSourceLocation());
+ unnestMapOp.setSourceLocation(dataSrcLoc);
unnestMapOp.getInputs().add(new MutableObject<>(inputOp));
context.computeAndSetTypeEnvironmentForOperator(unnestMapOp);
indexSearchOp = unnestMapOp;
@@ -800,6 +809,51 @@
return indexSearchOp;
}
+ private static ILogicalOperator addCastAssignOp(IOptimizationContext ctx,
List<IAType> indexKeysTypes,
+ BTreeJobGenParams jobGenParams, ILogicalOperator inputOp,
SourceLocation srcLoc)
+ throws AlgebricksException {
+ // cast the input values (low/high vars) if needed and update the
jobGenParams
+ List<LogicalVariable> lowKeyVars = jobGenParams.getLowKeyVarList();
+ List<LogicalVariable> highKeyVars = jobGenParams.getHighKeyVarList();
+ List<LogicalVariable> castAssignVars = new ArrayList<>();
+ List<Mutable<ILogicalExpression>> castAssignExprs = new ArrayList<>();
+ castInputValues(ctx, indexKeysTypes, lowKeyVars, inputOp,
castAssignVars, castAssignExprs, srcLoc);
+ castInputValues(ctx, indexKeysTypes, highKeyVars, inputOp,
castAssignVars, castAssignExprs, srcLoc);
+ if (castAssignVars.isEmpty()) {
+ return inputOp;
+ }
+ AssignOperator castAssignOp = new AssignOperator(castAssignVars,
castAssignExprs);
+ castAssignOp.setSourceLocation(srcLoc);
+ castAssignOp.getInputs().add(new MutableObject<>(inputOp));
+ castAssignOp.setExecutionMode(inputOp.getExecutionMode());
+ return castAssignOp;
+ }
+
+ private static void castInputValues(IOptimizationContext ctx, List<IAType>
indexKeyTypes,
+ List<LogicalVariable> inputVars, ILogicalOperator inputOp,
List<LogicalVariable> castAssignVars,
+ List<Mutable<ILogicalExpression>> castAssignExprs, SourceLocation
srcLoc) throws AlgebricksException {
+ for (int i = 0; i < inputVars.size(); i++) {
+ LogicalVariable inputVar = inputVars.get(i);
+ IVariableTypeEnvironment typeEnv =
ctx.getOutputTypeEnvironment(inputOp);
+ IAType varType = (IAType) typeEnv.getVarType(inputVar);
+ IAType varActualType = TypeComputeUtils.getActualType(varType);
+ IAType indexKeyType = indexKeyTypes.get(i);
+ if (varActualType.getTypeTag().isDerivedType() &&
indexKeyType.getTypeTag() == ATypeTag.ANY) {
+ LogicalVariable newInputVar = ctx.newVar();
+ castAssignVars.add(newInputVar);
+ VariableReferenceExpression newInputVarRef = new
VariableReferenceExpression(inputVar);
+ newInputVarRef.setSourceLocation(srcLoc);
+ ScalarFunctionCallExpression castFunc = new
ScalarFunctionCallExpression(
+
BuiltinFunctions.getBuiltinFunctionInfo(BuiltinFunctions.CAST_TYPE),
+ new ArrayList<>(Collections.singletonList(new
MutableObject<>(newInputVarRef))));
+ castFunc.setSourceLocation(srcLoc);
+ TypeCastUtils.setRequiredAndInputTypes(castFunc,
BuiltinType.ANY, varType);
+ castAssignExprs.add(new MutableObject<>(castFunc));
+ inputVars.set(i, newInputVar);
+ }
+ }
+ }
+
private int createKeyVarsAndExprs(int numKeys, LimitType[] keyLimits,
ILogicalExpression[] searchKeyExprs,
ArrayList<LogicalVariable> assignKeyVarList,
ArrayList<Mutable<ILogicalExpression>> assignKeyExprList,
ArrayList<LogicalVariable> keyVarList, IOptimizationContext
context, ILogicalExpression[] constExpressions,
diff --git
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryIndexOperationsHelper.java
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryIndexOperationsHelper.java
index cf4f8e0..7e1fd48 100644
---
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryIndexOperationsHelper.java
+++
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryIndexOperationsHelper.java
@@ -329,9 +329,10 @@
throws AlgebricksException {
IFunctionManager funManger = metadataProvider.getFunctionManager();
IDataFormat dataFormat = metadataProvider.getDataFormat();
- //if the target type is "BuiltinType.ANY" there is no need to cast. If
not we have to cast.
if (ATypeTag.ANY.equals(targetType.getTypeTag())) {
- return fieldEvalFactory;
+ // this is to ensure records and lists values are in the open
format
+ IScalarEvaluatorFactory[] castArg = new IScalarEvaluatorFactory[]
{ fieldEvalFactory };
+ return createCastFunction(targetType, BuiltinType.ANY, true,
sourceLoc).createEvaluatorFactory(castArg);
}
// check IndexUtil.castDefaultNull(index), too, because we always want
to cast even if the overriding type is
--
To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/19877
To unsubscribe, or for help writing mail filters, visit
https://asterix-gerrit.ics.uci.edu/settings
Gerrit-Project: asterixdb
Gerrit-Branch: ionic
Gerrit-Change-Id: I99f46564bc0879f2a82b0cd7dfbbd95132be7512
Gerrit-Change-Number: 19877
Gerrit-PatchSet: 1
Gerrit-Owner: Ali Alsuliman <[email protected]>
Gerrit-MessageType: newchange
