viirya commented on code in PR #226:
URL: https://github.com/apache/datafusion-comet/pull/226#discussion_r1575681660
##########
spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala:
##########
@@ -1646,6 +1647,37 @@ object QueryPlanSerde extends Logging with
ShimQueryPlanSerde {
None
}
+ case Murmur3Hash(children, seed) if children.forall(c =>
supportedDataType(c.dataType)) =>
+ // TODO: support list/map/struct type for murmur3 hash
+ val exprs = children.map(exprToProtoInternal(_, inputs))
+ val seedBuilder = ExprOuterClass.Literal
+ .newBuilder()
+ .setDatatype(serializeDataType(IntegerType).get)
+ .setIntVal(seed)
+ val seedExpr =
Some(ExprOuterClass.Expr.newBuilder().setLiteral(seedBuilder).build())
+ // the seed is put at the end of the arguments
+ scalarExprToProtoWithReturnType("murmur3_hash", IntegerType, exprs
:+ seedExpr: _*)
+
+ case Sha2(left, numBits) if numBits.foldable =>
+ // it's possible for spark to dynamically compute the number of bits
from input
+ // expression, however DataFusion does not support that yet.
+ val childExpr = exprToProtoInternal(left, inputs)
+ val bits = numBits.eval().asInstanceOf[Int]
+ val algorithm = bits match {
+ case 224 => "sha224"
+ case 256 | 0 => "sha256"
+ case 384 => "sha384"
+ case 512 => "sha512"
+ case _ =>
+ null
+ }
+ if (algorithm == null) {
+ emitWarning(s"numBits evaluated to a unknown value: $bits")
Review Comment:
This follows Spark behavior, seems no need to log warning message?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: github-unsubscr...@datafusion.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: github-unsubscr...@datafusion.apache.org
For additional commands, e-mail: github-h...@datafusion.apache.org
