korlov42 commented on a change in pull request #9276:
URL: https://github.com/apache/ignite/pull/9276#discussion_r680079886
##########
File path:
modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/metadata/IgniteMdSelectivity.java
##########
@@ -109,7 +149,612 @@ public Double getSelectivity(IgniteSortedIndexSpool rel,
RelMetadataQuery mq, Re
return mq.getSelectivity(rel.getInput(), rel.condition());
}
- /** */
+ public Double getSelectivity(RelSubset rel, RelMetadataQuery mq, RexNode
predicate) {
+ RelNode best = rel.getBest();
+ if (best == null)
+ return super.getSelectivity(rel, mq, predicate);
+
+ return getSelectivity(best, mq, predicate);
+ }
+
+ public Double getSelectivity(IgniteIndexScan rel, RelMetadataQuery mq,
RexNode predicate) {
+ return getTablePredicateBasedSelectivity(rel,
rel.getTable().unwrap(IgniteTable.class), mq, predicate);
+ }
+
+ public Double getSelectivity(IgniteTableScan rel, RelMetadataQuery mq,
RexNode predicate) {
+ IgniteTable tbl = rel.getTable().unwrap(IgniteTable.class);
+
+ return getTablePredicateBasedSelectivity(rel, tbl, mq, predicate);
+ }
+
+ /**
+ * Convert specified value into comparable type: BigDecimal,
+ *
+ * @param value Value to convert to comparable form.
+ * @return Comparable form of value.
+ */
+ private BigDecimal toComparableValue(RexLiteral value) {
+ RelDataType type = value.getType();
+ if (type instanceof BasicSqlType) {
+ BasicSqlType bType = (BasicSqlType)type;
+
+ switch ((SqlTypeFamily)bType.getFamily()) {
+ case NULL:
+ return null;
+
+ case NUMERIC:
+ return value.getValueAs(BigDecimal.class);
+
+ case DATE:
+ return new
BigDecimal(value.getValueAs(DateString.class).getMillisSinceEpoch());
+
+ case TIME:
+ return new
BigDecimal(value.getValueAs(TimeString.class).getMillisOfDay());
+
+ case TIMESTAMP:
+ return new
BigDecimal(value.getValueAs(TimestampString.class).getMillisSinceEpoch());
+
+ case BOOLEAN:
+ return (value.getValueAs(Boolean.class)) ? BigDecimal.ONE
: BigDecimal.ZERO;
+
+ }
+ }
+
+ //getOperand(value, );
+ return null;
+ }
+
+ /**
+ * Convert specified value into comparable type: BigDecimal,
+ *
+ * @param value Value to convert to comparable form.
+ * @return Comparable form of value.
+ */
+ private BigDecimal toComparableValue(Value value) {
+ if (value == null)
+ return null;
+
+ switch (value.getType()) {
+ case Value.NULL:
+ throw new IllegalArgumentException("Can't compare null
values");
+
+ case Value.BOOLEAN:
+ return (value.getBoolean()) ? BigDecimal.ONE : BigDecimal.ZERO;
+
+ case Value.BYTE:
+ return new BigDecimal(value.getByte());
+
+ case Value.SHORT:
+ return new BigDecimal(value.getShort());
+
+ case Value.INT:
+ return new BigDecimal(value.getInt());
+
+ case Value.LONG:
+ return new BigDecimal(value.getLong());
+
+ case Value.DECIMAL:
+ return value.getBigDecimal();
+
+ case Value.DOUBLE:
+ return new BigDecimal(value.getDouble());
+
+ case Value.FLOAT:
+ return new BigDecimal(value.getFloat());
+
+ case Value.DATE:
+ return new BigDecimal(value.getDate().getTime());
+
+ case Value.TIME:
+ return new BigDecimal(value.getTime().getTime());
+
+ case Value.TIMESTAMP:
+ return new BigDecimal(value.getTimestamp().getTime());
+
+ case Value.BYTES:
+ BigInteger bigInteger = new BigInteger(1, value.getBytes());
+ return new BigDecimal(bigInteger);
+
+ case Value.STRING:
+ case Value.STRING_FIXED:
+ case Value.STRING_IGNORECASE:
+ case Value.ARRAY:
+ case Value.JAVA_OBJECT:
+ case Value.GEOMETRY:
+ return null;
+
+ case Value.UUID:
+ BigInteger bigInt = new BigInteger(1, value.getBytes());
+ return new BigDecimal(bigInt);
+
+ default:
+ throw new IllegalStateException("Unsupported H2 type: " +
value.getType());
+ }
+ }
+
+ /**
+ * Predicate based selectivity for table. Estimate condition on each
column taking in comparison it's statistics.
+ *
+ * @param rel Original rel node to fallback calculation by.
+ * @param tbl Underlying IgniteTable.
+ * @param mq RelMetadataQuery
+ * @param predicate Predicate to estimate selectivity by.
+ * @return Selectivity.
+ */
+ private double getTablePredicateBasedSelectivity(RelNode rel, IgniteTable
tbl, RelMetadataQuery mq, RexNode predicate) {
+ if (tbl == null)
+ return super.getSelectivity(rel, mq, predicate);
+
+ double sel = 1.0;
+
+ Map<ColumnStatistics, Boolean> addNotNull = new HashMap<>();
+
+ for (RexNode pred : RelOptUtil.conjunctions(predicate)) {
+ ColumnStatistics colStat = getColStat(rel, mq, tbl, pred);
+
+ if (pred.getKind() == SqlKind.IS_NULL)
+ sel *= estimateIsNullSelectivity(rel, mq, tbl, pred);
+ else if (pred.getKind() == SqlKind.IS_NOT_NULL) {
+ if (colStat != null)
+ addNotNull.put(colStat, Boolean.FALSE);
+
+ sel *= estimateIsNotNullSelectivity(rel, mq, tbl, pred);
+ } else if (pred.isA(SqlKind.EQUALS)) {
+ if (colStat != null) {
+ Boolean colNotNull = addNotNull.get(colStat);
+
+ addNotNull.put(colStat, (colNotNull == null) ||
colNotNull);
+ }
+
+ sel *= estimateEqualsSelectivity(rel, mq, tbl, pred);
+ } else if (pred.isA(SqlKind.COMPARISON)) {
+ if (colStat != null) {
+ Boolean colNotNull = addNotNull.get(colStat);
+
+ addNotNull.put(colStat, (colNotNull == null) ||
colNotNull);
+ }
+
+ sel *= estimateComparisonSelectivity(rel, mq, tbl, pred);
+ } else
+ sel *= .25;
+ }
+
+ // Estimate not null selectivity in addition to comparison.
+ for (Map.Entry<ColumnStatistics, Boolean> colAddNotNull :
addNotNull.entrySet())
+ if (colAddNotNull.getValue())
+ sel *= estimateNotNullSelectivity(colAddNotNull.getKey());
+
+ return sel;
+ }
+
+ /**
+ * Compute selectivity for "is null" condition.
+ *
+ * @param rel RelNode.
+ * @param mq RelMetadataQuery.
+ * @param tbl IgniteTable.
+ * @param pred RexNode.
+ * @return Selectivity estimation.
+ */
+ private double estimateIsNullSelectivity(RelNode rel, RelMetadataQuery mq,
IgniteTable tbl, RexNode pred) {
+ ColumnStatistics colStat = getColStat(rel, mq, tbl, pred);
+
+ if (colStat == null)
+ return guessSelectivity(pred);
+
+ return estimateNullSelectivity(colStat);
+ }
+
+ /**
+ * Compute selectivity for "is not null" condition.
+ *
+ * @param rel RelNode.
+ * @param mq RelMetadataQuery.
+ * @param tbl IgniteTable.
+ * @param pred RexNode.
+ * @return Selectivity estimation.
+ */
+ private double estimateIsNotNullSelectivity(RelNode rel, RelMetadataQuery
mq, IgniteTable tbl, RexNode pred) {
+ ColumnStatistics colStat = getColStat(rel, mq, tbl, pred);
+
+ if (colStat == null)
+ return guessSelectivity(pred);
+
+ return estimateNotNullSelectivity(colStat);
+ }
+
+ /**
+ * Estimate selectivity for equals predicate.
+ *
+ * @param rel RElNode.
+ * @param mq RelMetadataQuery.
+ * @param tbl IgniteTable.
+ * @param pred RexNode with predicate.
+ *
+ * @return Selectivity.
+ */
+ private double estimateEqualsSelectivity(
+ RelNode rel,
+ RelMetadataQuery mq,
+ IgniteTable tbl,
+ RexNode pred) {
+ ColumnStatistics colStat = getColStat(rel, mq, tbl, pred);
+
+ if (colStat == null)
+ return guessSelectivity(pred);
+
+ RexLiteral val = getOperand(pred, RexLiteral.class);
+
+ if (val == null)
+ return guessSelectivity(pred);
+
+ BigDecimal comparableVal = toComparableValue(val);
+
+ if (comparableVal == null)
+ return guessSelectivity(pred);
+
+ return estimateEqualsSelectivity(colStat, comparableVal);
+ }
+
+ /**
+ * Estimate selectivity for comparison predicate.
+ *
+ * @param rel RelNode.
+ * @param mq RelMetadataQuery.
+ * @param tbl IgniteTable.
+ * @param pred RexNode.
+ * @return Selectivity.
+ */
+ private double estimateComparisonSelectivity(RelNode rel, RelMetadataQuery
mq, IgniteTable tbl, RexNode pred) {
+ ColumnStatistics colStat = getColStat(rel, mq, tbl, pred);
+
+ if (colStat == null)
+ return guessSelectivity(pred);
+
+ return estimateRangeSelectivity(colStat, pred);
+ }
+
+ /**
+ * Get column statistics.
+ *
+ * @param rel RelNode.
+ * @param mq RelMetadataQuery.
+ * @param tbl IgniteTable to get statistics from.
+ * @param pred Predicate to get statistics by related column.
+ * @return ColumnStatistics or {@code null}.
+ */
+ private ColumnStatistics getColStat(RelNode rel, RelMetadataQuery mq,
IgniteTable tbl, RexNode pred) {
+ Statistic stat = tbl.getStatistic();
+
+ if (stat == null)
+ return null;
+
+ RexNode operand = getOperand(pred, RexLocalRef.class);
+
+ if (operand == null)
+ return null;
+
+ Set<RelColumnOrigin> origins = mq.getColumnOrigins(rel,
((RexSlot)operand).getIndex());
+
+ if (origins == null || origins.isEmpty())
+ return null;
+
+ IgniteTypeFactory typeFactory = Commons.typeFactory(rel);
+
+ List<String> columns = tbl.getRowType(typeFactory).getFieldNames();
+
+ String colName =
columns.get(origins.iterator().next().getOriginColumnOrdinal());
+
+ // String colName = getColName(pred, columns);
+
+ if (QueryUtils.KEY_FIELD_NAME.equals(colName))
+ colName = tbl.descriptor().typeDescription().keyFieldName();
+
+ return
((IgniteStatisticsImpl)stat).getColumnsStatistics().get(colName);
+ }
+
+ /**
+ * Estimate range selectivity based on predicate.
+ *
+ * @param colStat Column statistics to use.
+ * @param pred Condition.
+ * @return Selectivity.
+ */
+ private double estimateRangeSelectivity(ColumnStatistics colStat, RexNode
pred) {
+ if (pred instanceof RexCall) {
+ RexLiteral literal = getOperand(pred, RexLiteral.class);
+ BigDecimal val = toComparableValue(literal);
+
+ return estimateSelectivity(colStat, val, pred);
+ }
+
+ return 1.;
+ }
+
+ /**
+ * Estimate range selectivity based on predicate, condition and column
statistics.
+ *
+ * @param colStat Column statistics to use.
+ * @param val Condition value.
+ * @param pred Condition.
+ * @return Selectivity.
+ */
+ private double estimateSelectivity(ColumnStatistics colStat, BigDecimal
val, RexNode pred) {
+ // Without value or statistics we can only guess.
+ if (val == null)
+ return guessSelectivity(pred);
+
+ SqlOperator op = ((RexCall)pred).op;
+
+ BigDecimal min = toComparableValue(colStat.min());
+ BigDecimal max = toComparableValue(colStat.max());
+ BigDecimal total = (min == null || max == null) ? null :
max.subtract(min).abs();
+
+ if (total == null)
+ // No min/max mean that all values are null for coumn.
+ return 0.;
+
+ // All values the same so check condition and return all or nothing
selectivity.
+ if (total.signum() == 0) {
+ BigDecimal diff = val.subtract(min);
+ int diffSign = diff.signum();
+
+ switch (op.getKind()) {
+ case GREATER_THAN:
+ return (diffSign < 0) ? 1. : 0.;
+
+ case LESS_THAN:
+ return (diffSign > 0) ? 1. : 0.;
+
+ case GREATER_THAN_OR_EQUAL:
+ return (diffSign <= 0) ? 1. : 0.;
+
+ case LESS_THAN_OR_EQUAL:
+ return (diffSign >= 0) ? 1. : 0.;
+
+ default:
+ return guessSelectivity(pred);
+ }
+ }
+
+ // Estimate percent of selectivity by ranges.
+ BigDecimal actual = BigDecimal.ZERO;
+
+ switch (op.getKind()) {
+ case GREATER_THAN:
+ case GREATER_THAN_OR_EQUAL:
+ actual = max.subtract(val);
+
+ if (actual.signum() < 0)
+ return 0.;
+
+ break;
+
+ case LESS_THAN:
+ case LESS_THAN_OR_EQUAL:
+ actual = val.subtract(min);
+
+ if (actual.signum() < 0)
+ return 0.;
+
+ break;
+
+ default:
+ return guessSelectivity(pred);
+ }
+
+ return (actual.compareTo(total) > 0) ? 1 : actual.divide(total,
MATH_CONTEXT).doubleValue();
+ }
+
+ /**
+ * Estimate "=" selectivity by column statistics.
+ *
+ * @param colStat Column statistics.
+ * @param comparableVal Comparable value to compare with.
+ * @return Selectivity.
+ */
+ private double estimateEqualsSelectivity(ColumnStatistics colStat,
BigDecimal comparableVal) {
+ if (colStat.total() == 0)
+ return 1.;
+
+ if (colStat.total() - colStat.nulls() == 0)
+ return 0.;
+
+ if (colStat.min() != null) {
+ BigDecimal minComparable = toComparableValue(colStat.min());
+ if (minComparable != null &&
minComparable.compareTo(comparableVal) > 0)
+ return 0.;
+ }
+
+ if (colStat.max() != null) {
+ BigDecimal maxComparable = toComparableValue(colStat.max());
+ if (maxComparable != null &&
maxComparable.compareTo(comparableVal) < 0)
+ return 0.;
+ }
+
+ double expectedRows = (double)(colStat.distinct()) / (colStat.total()
- colStat.nulls());
+
+ return expectedRows / colStat.total();
+ }
+
+ /**
+ * Estimate "is not null" selectivity by column statistics.
+ *
+ * @param colStat Column statistics.
+ * @return Selectivity.
+ */
+ private double estimateNotNullSelectivity(ColumnStatistics colStat) {
+ if (colStat.total() == 0)
+ return 1;
+
+ return (double)(colStat.total() - colStat.nulls()) / colStat.total();
+ }
+
+ /**
+ * Estimate "is null" selectivity by column statistics.
+ *
+ * @param colStat Column statistics.
+ * @return Selectivity.
+ */
+ private double estimateNullSelectivity(ColumnStatistics colStat) {
+ if (colStat.total() == 0)
+ return 1;
+
+ return (double)colStat.nulls() / colStat.total();
+ }
+
+ /**
+ * Get RexNode operand by type.
+ *
+ * @param pred RexNode to get operand by.
+ * @param cls Operand class.
+ * @return Operand or {@code null} if it unable to find operand with
specified type.
+ */
+ private <T> T getOperand(RexNode pred, Class<T> cls) {
+ if (pred instanceof RexCall) {
+ List<RexNode> operands = ((RexCall)pred).getOperands();
+
+ if (operands.size() > 2)
+ return null;
+
+ for (RexNode operand : operands) {
+ if (cls.isAssignableFrom(operand.getClass()))
+ return (T)operand;
+
+ if (operand instanceof RexCall) {
+ T res = getOperand(operand, cls);
+
+ if (res != null)
+ return res;
+ }
+ }
+ }
+ return null;
+ }
+
+ /**
+ * Guess selectivity by predicate type only.
+ *
+ * @param pred Predicate to guess selectivity by.
+ * @return Selectivity.
+ */
+ private double guessSelectivity(RexNode pred) {
+ if (pred.getKind() == SqlKind.IS_NULL)
+ return .1;
+ else if (pred.getKind() == SqlKind.IS_NOT_NULL)
+ return .9;
+ else if (pred.isA(SqlKind.EQUALS))
+ return .15;
+ else if (pred.isA(SqlKind.COMPARISON))
+ return .5;
+ else
+ return .25;
+ }
+
+ /**
+ * Get selectivity of exchange by it's input selectivity.
+ *
+ * @param exch IgniteExchange.
+ * @param mq RelMetadataQuery
+ * @param predicate Predicate.
+ * @return Selectivity or {@code null} if it can't be estimated.
+ */
+ public Double getSelectivity(IgniteExchange exch, RelMetadataQuery mq,
RexNode predicate) {
+ RelNode input = exch.getInput();
+
+ if (input == null)
+ return null;
+
+ return getSelectivity(input, mq, predicate);
+ }
+
+ /**
+ * Get selectivity of table spool by it's input selectivity.
+ *
+ * @param tspool IgniteTableSpool.
+ * @param mq RelMetadataQuery
+ * @param predicate Predicate.
+ * @return Selectivity or {@code null} if it can't be estimated.
+ */
+ public Double getSelectivity(IgniteTableSpool tspool, RelMetadataQuery mq,
RexNode predicate) {
+ RelNode input = tspool.getInput();
+
+ if (input == null)
+ return null;
+
+ return getSelectivity(input, mq, predicate);
+ }
+
+ /**
+ * Get selectivity of join by it's input selectivity.
+ *
+ * @param join AbstractIgniteJoin.
+ * @param mq RelMetadataQuery
+ * @param predicate Predicate.
+ * @return Selectivity or {@code null} if it can't be estimated.
+ */
+ public Double getSelectivity(AbstractIgniteJoin join, RelMetadataQuery mq,
RexNode predicate) {
+ double sel = 1.0;
+ JoinRelType joinType = join.getJoinType();
+ RexNode leftPred, rightPred;
+ final RexBuilder rexBuilder = join.getCluster().getRexBuilder();
+ int[] adjustments = new int[join.getRowType().getFieldCount()];
+
+ if (predicate != null) {
+ RexNode pred;
+ List<RexNode> leftFilters = new ArrayList<>();
+ List<RexNode> rightFilters = new ArrayList<>();
+ List<RexNode> joinFilters = new ArrayList<>();
+ List<RexNode> predList = RelOptUtil.conjunctions(predicate);
+
+ RelOptUtil.classifyFilters(
+ join,
+ predList,
+ joinType,
+ joinType == JoinRelType.INNER,
+ !joinType.generatesNullsOnLeft(),
+ !joinType.generatesNullsOnRight(),
+ joinFilters,
+ leftFilters,
+ rightFilters);
+ leftPred =
+ RexUtil.composeConjunction(rexBuilder, leftFilters, true);
+ rightPred =
+ RexUtil.composeConjunction(rexBuilder, rightFilters, true);
+
+ for (RelNode child : join.getInputs()) {
+ RexNode modifiedPred = null;
+
+ if (child == join.getLeft())
+ pred = leftPred;
+ else
+ pred = rightPred;
+
+ if (pred != null) {
+ // convert the predicate to reference the types of the
children
+ modifiedPred =
+ pred.accept(new RelOptUtil.RexInputConverter(
+ rexBuilder,
+ null,
+ child.getRowType().getFieldList(),
+ adjustments));
+ }
+ sel *= mq.getSelectivity(child, modifiedPred);
Review comment:
possible NPE here
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
