Github user viirya commented on a diff in the pull request:
https://github.com/apache/spark/pull/10435#discussion_r48528123
--- Diff:
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/misc.scala
---
@@ -176,3 +179,223 @@ case class Crc32(child: Expression) extends
UnaryExpression with ImplicitCastInp
})
}
}
+
+/**
+ * A function that calculates hash value for a group of expressions.
+ *
+ * The hash value for an expression depends on its type:
+ * - null: 0
+ * - boolean: 0 for true, 1 for false.
+ * - byte, short, int: the input itself.
+ * - long: input XOR (input >>> 32)
+ * - float: java.lang.Float.floatToIntBits(input)
+ * - double: l = java.lang.Double.doubleToLongBits(input); l
XOR (l >>> 32)
+ * - binary: java.util.Arrays.hashCode(input)
+ * - array: recursively calculate hash value for each
element, and aggregate them by
+ * `result = result * 37 + elementHash` with an
initial value `result = 37`.
+ * - map: recursively calculate hash value for each
key-value pair, and aggregate
+ * them by `result += keyHash XOR valueHash`.
+ * - struct: similar to array, calculate hash value for each
field and aggregate them.
+ * - other type: input.hashCode().
+ * e.g. calculate hash value for string type by
`UTF8String.hashCode()`.
+ * Finally we aggregate the hash values for each expression by the same
way of array.
+ */
+case class Hash(children: Seq[Expression]) extends Expression {
+
+ override def dataType: DataType = IntegerType
+
+ override def foldable: Boolean = children.forall(_.foldable)
+
+ override def nullable: Boolean = false
+
+ override def checkInputDataTypes(): TypeCheckResult = {
+ if (children.isEmpty) {
+ TypeCheckResult.TypeCheckFailure("input to function hash cannot be
empty")
+ } else {
+ TypeCheckResult.TypeCheckSuccess
+ }
+ }
+
+ override def eval(input: InternalRow): Any = {
+ var result = 37
+ for (e <- children) {
+ val hashValue = computeHash(e.eval(input), e.dataType)
+ result = result * 37 + hashValue
+ }
+ result
+ }
+
+ private def computeHash(v: Any, dataType: DataType): Int = v match {
+ case null => 0
+ case b: Boolean => if (b) 0 else 1
+ case b: Byte => b.toInt
+ case s: Short => s.toInt
+ case i: Int => i
+ case l: Long => (l ^ (l >>> 32)).toInt
+ case f: Float => java.lang.Float.floatToIntBits(f)
+ case d: Double =>
+ val b = java.lang.Double.doubleToLongBits(d)
+ (b ^ (b >>> 32)).toInt
+ case a: Array[Byte] => java.util.Arrays.hashCode(a)
+
+ case array: ArrayData =>
+ val elementType = dataType.asInstanceOf[ArrayType].elementType
+ var result = 0
+ var i = 0
+ while (i < array.numElements()) {
+ val hashValue = computeHash(array.get(i, elementType), elementType)
+ result = result * 37 + hashValue
--- End diff --
37? Looks like Hive uses 31.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at [email protected] or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
