>From <[email protected]>:
[email protected] has uploaded this change for review. (
https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17946 )
Change subject: [ASTERIXDB-3303][COMP] projection sizes
......................................................................
[ASTERIXDB-3303][COMP] projection sizes
Change-Id: I0f9df7b7937e0248c6fdae84a527680fac2fa915
---
M
asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/cbo/Stats.java
M
asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/cbo/JoinEnum.java
M
asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/BuiltinFunctions.java
3 files changed, 151 insertions(+), 16 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb
refs/changes/46/17946/1
diff --git
a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/cbo/JoinEnum.java
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/cbo/JoinEnum.java
index d33f8ce..47eed0f 100644
---
a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/cbo/JoinEnum.java
+++
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/cbo/JoinEnum.java
@@ -38,8 +38,10 @@
import org.apache.asterix.metadata.declared.MetadataProvider;
import org.apache.asterix.metadata.declared.SampleDataSource;
import org.apache.asterix.metadata.entities.Index;
+import org.apache.asterix.om.base.ADouble;
import org.apache.asterix.om.base.AInt64;
import org.apache.asterix.om.base.AOrderedList;
+import org.apache.asterix.om.base.ARecord;
import org.apache.asterix.om.base.IAObject;
import org.apache.asterix.om.constants.AsterixConstantValue;
import org.apache.asterix.om.functions.BuiltinFunctions;
@@ -836,6 +838,20 @@
return null;
}
+ // do I need to check for nulls being returned?
+ // Can have null returned, so this routine should only be called if at
least tuple is returned by the sample
+ private double findProjectedSize(ARecord record) {
+ // Now figure out the projected size
+ double projectedSize = 0.0;
+ int fields = record.numberOfFields();
+ for (int j = 1; j < fields; j++) {
+ IAObject field = record.getValueByPos(j);
+ double size = ((double) ((ADouble) field).getDoubleValue());
+ projectedSize += size;
+ }
+ return projectedSize;
+ }
+
// Most of this work is done in the very first line by calling
initializeBaseLevelJoinNodes().
// the remaining work here is to find the selectivities of the predicates
using sampling.
// By the time execution reaches this point, samples are guaranteed to
exist on all datasets,
@@ -898,7 +914,19 @@
// There are predicates here. So skip the predicates and
get the original dataset card.
// Now apply all the predicates and get the card after all
predicates are applied.
result = stats.runSamplingQuery(this.optCtx, leafInput);
- double predicateCardinality = ((double) ((AInt64)
result.get(0).get(0)).getLongValue());
+ //List<IAObject> result2 = result.get(0);
+ //IAObject result3 = result2.get(0);
+ //ARecord result4 = (ARecord) result3; record below
+
+ ARecord record = (ARecord) (((IAObject) ((List<IAObject>)
(result.get(0))).get(0)));
+ int fields = record.numberOfFields();
+ IAObject first = record.getValueByPos(0);
+ double predicateCardinality = ((double) ((AInt64)
first).getLongValue());
+
+ double projectedSize = -1.0;
+ if (predicateCardinality > 0.0) { // otherwise, we get
nulls for the averages
+ projectedSize = findProjectedSize(record);
+ }
if (predicateCardinality == 0.0) {
predicateCardinality = 0.0001 *
idxDetails.getSampleCardinalityTarget();
}
@@ -913,11 +941,15 @@
.warn(Warning.of(scanOp.getSourceLocation(), ErrorCode.SAMPLE_HAS_ZERO_ROWS));
}
}
-
finalDatasetCard *= predicateCardinality / sampleCard;
// now switch the input back.
parent.getInputs().get(0).setValue(scanOp);
jn.setCardinality(finalDatasetCard);
+ if (projectedSize > 0.0) {
+ jn.setAvgDocSize(projectedSize);
+ } else {
+ jn.setAvgDocSize(fields * 100); // cant think of
anything better... cards are more important anyway
+ }
}
}
dataScanPlan = jn.addSingleDatasetPlans();
@@ -1052,7 +1084,10 @@
SelectOperator selOp = new SelectOperator(new
MutableObject<>(exp));
selOp.getInputs().add(new MutableObject<>(leafInput));
result = stats.runSamplingQuery(this.optCtx, selOp);
- predicateCardinality = ((double) ((AInt64)
result.get(0).get(0)).getLongValue());
+ ARecord record = (ARecord) (((IAObject) ((List<IAObject>)
(result.get(0))).get(0)));
+ IAObject first = record.getValueByPos(0);
+ predicateCardinality = ((double) ((AInt64)
first).getLongValue());
+
if (predicateCardinality == 0.0) {
predicateCardinality = 0.0001 *
idxDetails.getSampleCardinalityTarget();
}
diff --git
a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/cbo/Stats.java
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/cbo/Stats.java
index d59fd73..f3c80ea 100644
---
a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/cbo/Stats.java
+++
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/cbo/Stats.java
@@ -24,6 +24,9 @@
import java.util.List;
import org.apache.asterix.compiler.provider.IRuleSetFactory;
+import org.apache.asterix.lang.common.expression.LiteralExpr;
+import org.apache.asterix.lang.common.literal.StringLiteral;
+import org.apache.asterix.lang.common.util.FunctionUtil;
import org.apache.asterix.metadata.declared.DataSource;
import org.apache.asterix.metadata.declared.DataSourceId;
import org.apache.asterix.metadata.declared.MetadataProvider;
@@ -31,10 +34,12 @@
import org.apache.asterix.metadata.entities.Index;
import org.apache.asterix.om.base.AInt64;
import org.apache.asterix.om.base.IAObject;
+import org.apache.asterix.om.constants.AsterixConstantValue;
import org.apache.asterix.om.functions.BuiltinFunctionInfo;
import org.apache.asterix.om.functions.BuiltinFunctions;
import org.apache.asterix.optimizer.base.AnalysisUtil;
import
org.apache.asterix.optimizer.rules.am.array.AbstractOperatorFromSubplanRewrite;
+import org.apache.asterix.translator.ConstantHelper;
import org.apache.commons.lang3.mutable.Mutable;
import org.apache.commons.lang3.mutable.MutableObject;
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
@@ -49,11 +54,17 @@
import
org.apache.hyracks.algebricks.core.algebra.expressions.ConstantExpression;
import
org.apache.hyracks.algebricks.core.algebra.expressions.JoinProductivityAnnotation;
import
org.apache.hyracks.algebricks.core.algebra.expressions.PredicateCardinalityAnnotation;
+import
org.apache.hyracks.algebricks.core.algebra.expressions.ScalarFunctionCallExpression;
+import
org.apache.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
import
org.apache.hyracks.algebricks.core.algebra.functions.AlgebricksBuiltinFunctions;
import
org.apache.hyracks.algebricks.core.algebra.operators.logical.AggregateOperator;
+import
org.apache.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
import
org.apache.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator;
+import
org.apache.hyracks.algebricks.core.algebra.operators.logical.ProjectOperator;
import
org.apache.hyracks.algebricks.core.algebra.operators.logical.SelectOperator;
import
org.apache.hyracks.algebricks.core.algebra.operators.logical.SubplanOperator;
+import
org.apache.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
+import org.apache.hyracks.algebricks.core.algebra.plan.ALogicalPlanImpl;
import
org.apache.hyracks.algebricks.core.algebra.util.OperatorManipulationUtil;
import org.apache.hyracks.algebricks.core.algebra.util.OperatorPropertiesUtil;
import org.apache.hyracks.api.exceptions.ErrorCode;
@@ -521,9 +532,6 @@
return sel;
}
- private void transformtoAnyInPlan(SelectOperator newSelOp) {
- }
-
protected List<List<IAObject>> runSamplingQuery(IOptimizationContext ctx,
ILogicalOperator logOp)
throws AlgebricksException {
LOGGER.info("***running sample query***");
@@ -531,28 +539,109 @@
IOptimizationContext newCtx =
ctx.getOptimizationContextFactory().cloneOptimizationContext(ctx);
ILogicalOperator newScanOp =
OperatorManipulationUtil.bottomUpCopyOperators(logOp);
+ // Now we have to generate plans like this on top of the scanOp (logOp)
+ // project ([$$79])
+ // assign [$$79] <- [{"$1": $$73, "$2": $$74, "$3": $$75, "$4": $$76,
"$5": $$77, "$6": $$78}]
+ // aggregate [$$73, $$74, $$75, $$76, $$77, $$78] <- [agg-count(true),
sql-avg($$68), sql-avg($$69), sql-avg($$70), sql-avg($$71), sql-avg($$72)]
+ // assign [$$68, $$69, $$70, $$71, $$72] <- [serialized-size($$60),
serialized-size($$str), serialized-size($$61), serialized-size($$65),
serialized-size($$67)]
+ // add the assign [$$56, ..., ] <- [encoded-size($$67), ..., ] on top
of newAggOp
+ List<LogicalVariable> vars = new ArrayList<>();
+ VariableUtilities.getLiveVariables(logOp, vars);
+ LogicalVariable newVar;
+ // array to keep track of the assigns
+ List<LogicalVariable> newVars = new ArrayList<>();
+ List<Mutable<ILogicalExpression>> exprs = new ArrayList<>();
+
+ // create this assignOperator
+ //assign [$$46, $$47, $$48, $$49, $$50] <- [serialized-size($$40),
serialized-size($$str), serialized-size($$41), serialized-size($$43),
serialized-size($$44)]
+ int count = 0;
+ for (LogicalVariable lv : vars) {
+ count++;
+ VariableReferenceExpression varRefExpr = new
VariableReferenceExpression(lv);
+ List<Mutable<ILogicalExpression>> vars2 = new ArrayList<>();
+ vars2.add(new MutableObject<>(varRefExpr));
+ ScalarFunctionCallExpression func = new
ScalarFunctionCallExpression(
+
FunctionUtil.getFunctionInfo(BuiltinFunctions.SERIALIZED_SIZE), vars2);
+ exprs.add(new MutableObject<>(func));
+ newVar = newCtx.newVar();
+ newVars.add(newVar);
+ }
+
+ AssignOperator assignOp = new AssignOperator(newVars, exprs);
+ assignOp.getInputs().add(new MutableObject<>(newScanOp));
+ Mutable<ILogicalOperator> tmpRef = new MutableObject<>(assignOp);
+ String viewInPlan = new ALogicalPlanImpl(tmpRef).toString();
+
+ // add the count-agg (true) first
+ List<LogicalVariable> newVars2 = new ArrayList<>();
+ List<Mutable<ILogicalExpression>> aggExprList = new ArrayList<>();
List<Mutable<ILogicalExpression>> aggFunArgs = new ArrayList<>(1);
aggFunArgs.add(new MutableObject<>(ConstantExpression.TRUE));
BuiltinFunctionInfo countFn =
BuiltinFunctions.getBuiltinFunctionInfo(BuiltinFunctions.COUNT);
AggregateFunctionCallExpression aggExpr = new
AggregateFunctionCallExpression(countFn, false, aggFunArgs);
-
- List<Mutable<ILogicalExpression>> aggExprList = new ArrayList<>(1);
aggExprList.add(new MutableObject<>(aggExpr));
-
- List<LogicalVariable> aggVarList = new ArrayList<>(1);
LogicalVariable aggVar = newCtx.newVar();
- aggVarList.add(aggVar);
+ newVars2.add(aggVar);
- AggregateOperator newAggOp = new AggregateOperator(aggVarList,
aggExprList);
- newAggOp.getInputs().add(new MutableObject<>(newScanOp));
+ // Now add the aggregate operator
+ //aggregate [$$51, $$52, $$53, $$54, $$55, $$56] <- [count-agg(true),
sql-avg($$46), sql-avg($$47), sql-avg($$48), sql-avg($$49), sql-avg($$50)]
+ for (int i = 0; i < count; i++) {
+ VariableReferenceExpression varRefExpr;
+ varRefExpr = new VariableReferenceExpression(newVars.get(i));
+ List<Mutable<ILogicalExpression>> vars2 = new ArrayList<>();
+ vars2.add(new MutableObject<>(varRefExpr));
+ BuiltinFunctionInfo avgFn =
BuiltinFunctions.getBuiltinFunctionInfo(BuiltinFunctions.SQL_AVG);
+ aggExpr = new AggregateFunctionCallExpression(avgFn, false, vars2);
+ newVar = newCtx.newVar();
+
+ newVars2.add(newVar);
+ aggExprList.add(new MutableObject<>(aggExpr));
+ }
+
+ AggregateOperator newAggOp = new AggregateOperator(newVars2,
aggExprList);
+ newAggOp.getInputs().add(new MutableObject<>(assignOp));
Mutable<ILogicalOperator> newAggOpRef = new MutableObject<>(newAggOp);
+ OperatorPropertiesUtil.typeOpRec(newAggOpRef, newCtx); // is this
really needed??
- OperatorPropertiesUtil.typeOpRec(newAggOpRef, newCtx);
+ //List<MutableObject> arr =
createMutableObjectArray(newAggOp.getExpressions());
+ List<MutableObject> arr =
createMutableObjectArray(newAggOp.getVariables());
+ AbstractFunctionCallExpression f = new ScalarFunctionCallExpression(
+
FunctionUtil.getFunctionInfo(BuiltinFunctions.OPEN_RECORD_CONSTRUCTOR));
+ for (int i = 0; i < arr.size(); i++) {
+ f.getArguments().add(arr.get(i));
+ }
+
+ newVar = newCtx.newVar();
+ assignOp = new AssignOperator(newVar, new MutableObject<>(f));
+ assignOp.getInputs().add(new MutableObject<>(newAggOp));
+ ProjectOperator pOp = new ProjectOperator(newVar);
+ pOp.getInputs().add(new MutableObject<>(assignOp));
+
+ Mutable<ILogicalOperator> Ref = new MutableObject<>(pOp);
LOGGER.info("***returning from sample query***");
- return AnalysisUtil.runQuery(newAggOpRef, Arrays.asList(aggVar),
newCtx, IRuleSetFactory.RuleSetKind.SAMPLING);
+ OperatorPropertiesUtil.typeOpRec(Ref, newCtx);
+ String viewInPlan3 = new ALogicalPlanImpl(Ref).toString(); //useful
when debugging
+ LOGGER.trace("viewInPlan3");
+ LOGGER.trace(viewInPlan3);
+ return AnalysisUtil.runQuery(Ref, Arrays.asList(newVar), newCtx,
IRuleSetFactory.RuleSetKind.SAMPLING);
+
+ }
+
+ private List<MutableObject> createMutableObjectArray(List<LogicalVariable>
vars) {
+ List<MutableObject> arr = new ArrayList<>();
+ for (int i = 0; i < vars.size(); i++) {
+ LiteralExpr le = new LiteralExpr();
+ StringLiteral value = new StringLiteral("$" + Integer.toString(i +
1)); // these start from 1
+ le.setValue(value);
+ AsterixConstantValue cValue = new
AsterixConstantValue(ConstantHelper.objectFromLiteral(le.getValue()));
+ ConstantExpression cExpr = new ConstantExpression(cValue);
+ arr.add(new MutableObject<>(cExpr));
+ arr.add(new MutableObject<>(new
VariableReferenceExpression(vars.get(i))));
+ }
+ return arr;
}
public long findDistinctCardinality(ILogicalOperator grpByDistinctOp)
throws AlgebricksException {
diff --git
a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/BuiltinFunctions.java
b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/BuiltinFunctions.java
index 6c621e0..ab5ad0b 100644
---
a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/BuiltinFunctions.java
+++
b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/BuiltinFunctions.java
@@ -2108,7 +2108,9 @@
addFunction(DECODE_DATAVERSE_NAME,
OrderedListOfAStringTypeComputer.INSTANCE_NULLABLE, true);
addPrivateFunction(COLLECTION_TO_SEQUENCE,
CollectionToSequenceTypeComputer.INSTANCE, true);
- addPrivateFunction(SERIALIZED_SIZE, AInt64TypeComputer.INSTANCE, true);
+ addFunction(SERIALIZED_SIZE, AInt64TypeComputer.INSTANCE, true);
+ // used by CBO's internal sampling queries for determining projection
sizes
+
// external lookup
addPrivateFunction(EXTERNAL_LOOKUP, AnyTypeComputer.INSTANCE, false);
--
To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17946
To unsubscribe, or for help writing mail filters, visit
https://asterix-gerrit.ics.uci.edu/settings
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Change-Id: I0f9df7b7937e0248c6fdae84a527680fac2fa915
Gerrit-Change-Number: 17946
Gerrit-PatchSet: 1
Gerrit-Owner: [email protected]
Gerrit-MessageType: newchange
