xudong963 commented on code in PR #15296:
URL: https://github.com/apache/datafusion/pull/15296#discussion_r2004911809
##########
datafusion/expr-common/src/statistics.rs:
##########
@@ -857,6 +857,143 @@ pub fn compute_variance(
ScalarValue::try_from(target_type)
}
+/// Merges two distributions into a single distribution that represents their
combined statistics.
+/// This creates a more general distribution that approximates the mixture of
the input distributions.
+pub fn merge_distributions(a: &Distribution, b: &Distribution) ->
Result<Distribution> {
+ let range_a = a.range()?;
+ let range_b = b.range()?;
+
+ // Determine data type and create combined range
+ let combined_range = if range_a.is_unbounded() || range_b.is_unbounded() {
+ Interval::make_unbounded(&range_a.data_type())?
+ } else {
+ // Take the widest possible range conservatively
+ let lower_a = range_a.lower();
+ let lower_b = range_b.lower();
+ let upper_a = range_a.upper();
+ let upper_b = range_b.upper();
+
+ let combined_lower = if lower_a.lt(lower_b) {
+ lower_a.clone()
+ } else {
+ lower_b.clone()
+ };
+
+ let combined_upper = if upper_a.gt(upper_b) {
+ upper_a.clone()
+ } else {
+ upper_b.clone()
+ };
+
+ Interval::try_new(combined_lower, combined_upper)?
+ };
+
+ // Calculate weights for the mixture distribution
Review Comment:
And also ping @kosiew , do you have any thoughts for the new way to compute
the weight?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: github-unsubscr...@datafusion.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: github-unsubscr...@datafusion.apache.org
For additional commands, e-mail: github-h...@datafusion.apache.org
