dylanhz commented on code in PR #27325: URL: https://github.com/apache/flink/pull/27325#discussion_r2650053011
########## flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/aggregate/WelfordM2AggFunction.java: ########## @@ -0,0 +1,236 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.runtime.functions.aggregate; + +import org.apache.flink.annotation.Internal; +import org.apache.flink.table.api.DataTypes; +import org.apache.flink.table.data.DecimalData; +import org.apache.flink.table.data.DecimalDataUtils; +import org.apache.flink.table.types.DataType; +import org.apache.flink.table.types.logical.LogicalType; + +import javax.annotation.Nullable; + +import java.util.Collections; +import java.util.List; +import java.util.Objects; + +import static org.apache.flink.table.types.utils.DataTypeUtils.toInternalDataType; + +/** + * Internal built-in WELFORD_M2 aggregate function to calculate the m2 term in Welford's online + * algorithm. This is a helper function for variance related functions rewritten. + * + * @see <a + * href="https://en.wikipedia.org/wiki/Algorithms_for_calculating_variance#Welford's_online_algorithm">Welford's + * online algorithm</a> + */ +@Internal +public abstract class WelfordM2AggFunction + extends BuiltInAggregateFunction<Double, WelfordM2AggFunction.WelfordM2Accumulator> { + + protected final transient DataType valueType; + + public WelfordM2AggFunction(LogicalType inputType) { + this.valueType = toInternalDataType(inputType); + } + + @Override + public List<DataType> getArgumentDataTypes() { + return Collections.singletonList(valueType); + } + + @Override + public DataType getAccumulatorDataType() { + return DataTypes.STRUCTURED( + WelfordM2Accumulator.class, + DataTypes.FIELD("n", DataTypes.BIGINT().notNull()), + DataTypes.FIELD("mean", DataTypes.DOUBLE().notNull()), + DataTypes.FIELD("m2", DataTypes.DOUBLE().notNull())); + } + + @Override + public DataType getOutputDataType() { + return DataTypes.DOUBLE(); + } + + // -------------------------------------------------------------------------------------------- + // Accumulator + // -------------------------------------------------------------------------------------------- + + /** Accumulator for WELFORD_M2. */ + public static class WelfordM2Accumulator { + + public long n = 0L; + public double mean = 0.0D; + public double m2 = 0.0D; + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null || getClass() != obj.getClass()) { + return false; + } + WelfordM2Accumulator that = (WelfordM2Accumulator) obj; + return n == that.n + && Double.compare(mean, that.mean) == 0 + && Double.compare(m2, that.m2) == 0; + } + + @Override + public int hashCode() { + return Objects.hash(n, mean, m2); + } + } + + @Override + public WelfordM2Accumulator createAccumulator() { + return new WelfordM2Accumulator(); + } + + public void resetAccumulator(WelfordM2Accumulator acc) { + acc.n = 0L; + acc.mean = 0.0D; + acc.m2 = 0.0D; + } + + @Override + public Double getValue(WelfordM2Accumulator acc) { + // Theoretically, acc.m2 should always be non-negative. + // But in practice it may be negative if records are out of order, which is different from Review Comment: We decided that this out-of-order case is beyond Flink's design scope. I’ve deleted the comments but retained the implementation as a safeguard to maintain robustness. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
