wjones127 commented on code in PR #7467:
URL: https://github.com/apache/arrow-datafusion/pull/7467#discussion_r1323653375
##########
datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs:
##########
@@ -39,13 +39,19 @@ use datafusion_expr::{
and, expr, lit, or, BinaryExpr, BuiltinScalarFunction, Case,
ColumnarValue, Expr,
Like, Volatility,
};
-use datafusion_physical_expr::{create_physical_expr,
execution_props::ExecutionProps};
+use datafusion_physical_expr::{
+ create_physical_expr, execution_props::ExecutionProps,
intervals::NullableInterval,
+};
use crate::simplify_expressions::SimplifyInfo;
+use crate::simplify_expressions::guarantees::GuaranteeRewriter;
+
/// This structure handles API for expression simplification
pub struct ExprSimplifier<S> {
info: S,
+ ///
Review Comment:
Filled that in.
##########
datafusion/physical-expr/src/intervals/interval_aritmetic.rs:
##########
@@ -686,6 +720,282 @@ fn calculate_cardinality_based_on_bounds(
}
}
+/// An [Interval] that also tracks null status using a boolean interval.
+///
+/// This represents values that may be in a particular range or be null.
+///
+/// # Examples
+///
+/// ```
+/// use datafusion_physical_expr::intervals::{Interval, NullableInterval};
+/// use datafusion_common::ScalarValue;
+///
+/// // [1, 2) U {NULL}
+/// NullableInterval::MaybeNull {
+/// values: Interval::make(Some(1), Some(2), (false, true)),
+/// };
+///
+/// // (0, ∞)
+/// NullableInterval::NotNull {
+/// values: Interval::make(Some(0), None, (true, true)),
+/// };
+///
+/// // {NULL}
+/// NullableInterval::Null;
+///
+/// // {4}
+/// NullableInterval::from(&ScalarValue::Int32(Some(4)));
+/// ```
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub enum NullableInterval {
+ /// The value is always null in this interval
+ ///
+ /// This is typed so it can be used in physical expressions, which don't do
+ /// type coercion.
+ Null { datatype: DataType },
+ /// The value may or may not be null in this interval. If it is non null
its value is within
+ /// the specified values interval
+ MaybeNull { values: Interval },
+ /// The value is definitely not null in this interval and is within values
+ NotNull { values: Interval },
+}
+
+impl Default for NullableInterval {
+ fn default() -> Self {
+ NullableInterval::MaybeNull {
+ values: Interval::default(),
+ }
+ }
+}
+
+impl Display for NullableInterval {
+ fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
+ match self {
+ Self::Null { .. } => write!(f, "NullableInterval: {{NULL}}"),
+ Self::MaybeNull { values } => {
+ write!(f, "NullableInterval: {} U {{NULL}}", values)
+ }
+ Self::NotNull { values } => write!(f, "NullableInterval: {}",
values),
+ }
+ }
+}
+
+impl From<&ScalarValue> for NullableInterval {
+ /// Create an interval that represents a single value.
+ fn from(value: &ScalarValue) -> Self {
+ if value.is_null() {
+ Self::Null {
+ datatype: value.get_datatype(),
+ }
+ } else {
+ Self::NotNull {
+ values: Interval::new(
+ IntervalBound::new(value.clone(), false),
+ IntervalBound::new(value.clone(), false),
+ ),
+ }
+ }
+ }
+}
+
+impl NullableInterval {
+ /// Get the values interval, or None if this interval is definitely null.
+ pub fn values(&self) -> Option<&Interval> {
+ match self {
+ Self::Null { .. } => None,
+ Self::MaybeNull { values } | Self::NotNull { values } =>
Some(values),
+ }
+ }
+
+ /// Get the data type
+ pub fn get_datatype(&self) -> Result<DataType> {
+ match self {
+ Self::Null { datatype } => Ok(datatype.clone()),
+ Self::MaybeNull { values } | Self::NotNull { values } => {
+ values.get_datatype()
+ }
+ }
+ }
+
+ /// Return true if the value is definitely true (and not null).
+ pub fn is_certainly_true(&self) -> bool {
+ match self {
+ Self::Null { .. } | Self::MaybeNull { .. } => false,
+ Self::NotNull { values } => values == &Interval::CERTAINLY_TRUE,
+ }
+ }
+
+ /// Return true if the value is definitely false (and not null).
+ pub fn is_certainly_false(&self) -> bool {
+ match self {
+ Self::Null { .. } => false,
+ Self::MaybeNull { .. } => false,
+ Self::NotNull { values } => values == &Interval::CERTAINLY_FALSE,
+ }
+ }
+
+ /// Perform logical negation on a boolean nullable interval.
+ fn not(&self) -> Result<Self> {
+ match self {
+ Self::Null { datatype } => Ok(Self::Null {
+ datatype: datatype.clone(),
+ }),
+ Self::MaybeNull { values } => Ok(Self::MaybeNull {
+ values: values.not()?,
+ }),
+ Self::NotNull { values } => Ok(Self::NotNull {
+ values: values.not()?,
+ }),
+ }
+ }
+
+ /// Apply the given operator to this interval and the given interval.
+ ///
+ /// # Examples
+ ///
+ /// ```
+ /// use datafusion_common::ScalarValue;
+ /// use datafusion_expr::Operator;
+ /// use datafusion_physical_expr::intervals::{Interval, NullableInterval};
+ ///
+ /// // 4 > 3 -> true
+ /// let lhs = NullableInterval::from(&ScalarValue::Int32(Some(4)));
+ /// let rhs = NullableInterval::from(&ScalarValue::Int32(Some(3)));
+ /// let result = lhs.apply_operator(&Operator::Gt, &rhs).unwrap();
+ /// assert_eq!(result,
NullableInterval::from(&ScalarValue::Boolean(Some(true))));
+ ///
+ /// // [1, 3) > NULL -> NULL
+ /// let lhs = NullableInterval::NotNull {
+ /// values: Interval::make(Some(1), Some(3), (false, true)),
+ /// };
+ /// let rhs = NullableInterval::from(&ScalarValue::Int32(None));
+ /// let result = lhs.apply_operator(&Operator::Gt, &rhs).unwrap();
+ /// assert_eq!(result.single_value(), Some(ScalarValue::Null));
+ ///
+ /// // [1, 3] > [2, 4] -> [false, true]
+ /// let lhs = NullableInterval::NotNull {
+ /// values: Interval::make(Some(1), Some(3), (false, false)),
+ /// };
+ /// let rhs = NullableInterval::NotNull {
+ /// values: Interval::make(Some(2), Some(4), (false, false)),
+ /// };
+ /// let result = lhs.apply_operator(&Operator::Gt, &rhs).unwrap();
+ /// // Both inputs are valid (non-null), so result must be non-null
+ /// assert_eq!(result, NullableInterval::NotNull {
+ /// // Uncertain whether inequality is true or false
+ /// values: Interval::UNCERTAIN,
+ /// });
+ ///
+ /// ```
+ pub fn apply_operator(&self, op: &Operator, rhs: &Self) -> Result<Self> {
+ match op {
+ Operator::IsDistinctFrom => {
+ let values = match (self, rhs) {
+ // NULL is distinct from NULL -> False
+ (Self::Null { .. }, Self::Null { .. }) =>
Interval::CERTAINLY_FALSE,
+ // x is distinct from y -> x != y,
+ // if at least one of them is never null.
+ (Self::NotNull { .. }, _) | (_, Self::NotNull { .. }) => {
+ let lhs_values = self.values();
+ let rhs_values = rhs.values();
+ match (lhs_values, rhs_values) {
+ (Some(lhs_values), Some(rhs_values)) => {
+ lhs_values.equal(rhs_values).not()?
+ }
+ (Some(_), None) | (None, Some(_)) =>
Interval::CERTAINLY_TRUE,
+ (None, None) => unreachable!("Null case handled
above"),
+ }
+ }
+ _ => Interval::UNCERTAIN,
+ };
+ // IsDistinctFrom never returns null.
+ Ok(Self::NotNull { values })
+ }
+ Operator::IsNotDistinctFrom => self
+ .apply_operator(&Operator::IsDistinctFrom, rhs)
+ .map(|i| i.not())?,
+ _ => {
+ if let (Some(left_values), Some(right_values)) =
+ (self.values(), rhs.values())
+ {
+ let values = apply_operator(op, left_values,
right_values)?;
+ match (self, rhs) {
+ (Self::NotNull { .. }, Self::NotNull { .. }) => {
+ Ok(Self::NotNull { values })
+ }
+ _ => Ok(Self::MaybeNull { values }),
+ }
+ } else if op.is_comparison_operator() {
+ Ok(Self::Null {
+ datatype: DataType::Boolean,
+ })
+ } else {
+ Ok(Self::Null {
+ datatype: self.get_datatype()?,
+ })
+ }
+ }
+ }
+ }
+
+ /// Determine if this interval contains the given interval. Returns a
boolean
+ /// interval that is [true, true] if this interval is a superset of the
+ /// given interval, [false, false] if this interval is disjoint from the
+ /// given interval, and [false, true] otherwise.
+ pub fn contains<T: Borrow<Self>>(&self, other: T) -> Result<Self> {
+ let rhs = other.borrow();
+ if let (Some(left_values), Some(right_values)) = (self.values(),
rhs.values()) {
+ let values = left_values.contains(right_values)?;
+ match (self, rhs) {
+ (Self::NotNull { .. }, Self::NotNull { .. }) => {
+ Ok(Self::NotNull { values })
+ }
+ _ => Ok(Self::MaybeNull { values }),
+ }
+ } else {
+ Ok(Self::Null {
+ datatype: DataType::Boolean,
+ })
+ }
+ }
+
+ /// If the interval has collapsed to a single value, return that value.
+ ///
+ /// Otherwise returns None.
+ ///
+ /// # Examples
+ ///
+ /// ```
+ /// use datafusion_common::ScalarValue;
+ /// use datafusion_physical_expr::intervals::{Interval, NullableInterval};
+ ///
+ /// let interval = NullableInterval::from(&ScalarValue::Int32(Some(4)));
+ /// assert_eq!(interval.single_value(), Some(ScalarValue::Int32(Some(4))));
+ ///
+ /// let interval = NullableInterval::from(&ScalarValue::Int32(None));
+ /// assert_eq!(interval.single_value(), Some(ScalarValue::Null));
+ ///
+ /// let interval = NullableInterval::MaybeNull {
+ /// values: Interval::make(Some(1), Some(4), (false, true)),
+ /// };
+ /// assert_eq!(interval.single_value(), None);
+ /// ```
+ pub fn single_value(&self) -> Option<ScalarValue> {
Review Comment:
Changed.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
