Repository: flink Updated Branches: refs/heads/master 33d3b767e -> d5de2bcf1
[FLINK-6926] [table] Add support for SHA-224, SHA-384, SHA-512 Project: http://git-wip-us.apache.org/repos/asf/flink/repo Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/5544ab5d Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/5544ab5d Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/5544ab5d Branch: refs/heads/master Commit: 5544ab5df5d805511a40f2e4007b3afee06d15af Parents: 33d3b76 Author: Michael Gendelman <[email protected]> Authored: Sat Jan 13 23:24:08 2018 +0200 Committer: Timo Walther <[email protected]> Committed: Tue May 15 11:34:33 2018 +0200 ---------------------------------------------------------------------- docs/dev/table/sql.md | 33 ++++++++++ docs/dev/table/tableApi.md | 69 +++++++++++++++++++- .../flink/table/api/scala/expressionDsl.scala | 21 ++++++ .../table/codegen/calls/FunctionGenerator.scala | 18 +++++ .../table/expressions/hashExpressions.scala | 39 +++++++++++ .../functions/sql/ScalarSqlFunctions.scala | 27 ++++++++ .../flink/table/validate/FunctionCatalog.scala | 9 ++- .../table/expressions/ScalarFunctionsTest.scala | 49 ++++++++++++-- .../table/expressions/SqlExpressionTest.scala | 13 ++++ 9 files changed, 269 insertions(+), 9 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/flink/blob/5544ab5d/docs/dev/table/sql.md ---------------------------------------------------------------------- diff --git a/docs/dev/table/sql.md b/docs/dev/table/sql.md index ce0e1d6..7f4a386 100644 --- a/docs/dev/table/sql.md +++ b/docs/dev/table/sql.md @@ -2437,6 +2437,17 @@ SHA1(string) <tr> <td> {% highlight text %} +SHA224(string) +{% endhighlight %} + </td> + <td> + <p>Returns the SHA-224 hash of the string argument as a string of 56 hexadecimal digits; null if <i>string</i> is null.</p> + </td> + </tr> + + <tr> + <td> + {% highlight text %} SHA256(string) {% endhighlight %} </td> @@ -2444,6 +2455,28 @@ SHA256(string) <p>Returns the SHA-256 hash of the string argument as a string of 64 hexadecimal digits; null if <i>string</i> is null.</p> </td> </tr> + + <tr> + <td> + {% highlight text %} +SHA384(string) +{% endhighlight %} + </td> + <td> + <p>Returns the SHA-384 hash of the string argument as a string of 96 hexadecimal digits; null if <i>string</i> is null.</p> + </td> + </tr> + + <tr> + <td> + {% highlight text %} +SHA512(string) +{% endhighlight %} + </td> + <td> + <p>Returns the SHA-512 hash of the string argument as a string of 128 hexadecimal digits; null if <i>string</i> is null.</p> + </td> + </tr> </tbody> </table> http://git-wip-us.apache.org/repos/asf/flink/blob/5544ab5d/docs/dev/table/tableApi.md ---------------------------------------------------------------------- diff --git a/docs/dev/table/tableApi.md b/docs/dev/table/tableApi.md index 69aaeea..05ce604 100644 --- a/docs/dev/table/tableApi.md +++ b/docs/dev/table/tableApi.md @@ -3083,7 +3083,18 @@ STRING.sha1() </td> </tr> - <tr> + <tr> + <td> + {% highlight java %} +STRING.sha224() +{% endhighlight %} + </td> + <td> + <p>Returns the SHA-224 hash of the string argument as a string of 56 hexadecimal digits; null if <i>string</i> is null.</p> + </td> + </tr> + + <tr> <td> {% highlight java %} STRING.sha256() @@ -3094,6 +3105,28 @@ STRING.sha256() </td> </tr> + <tr> + <td> + {% highlight java %} +STRING.sha384() +{% endhighlight %} + </td> + <td> + <p>Returns the SHA-384 hash of the string argument as a string of 96 hexadecimal digits; null if <i>string</i> is null.</p> + </td> + </tr> + + <tr> + <td> + {% highlight java %} +STRING.sha512() +{% endhighlight %} + </td> + <td> + <p>Returns the SHA-512 hash of the string argument as a string of 128 hexadecimal digits; null if <i>string</i> is null.</p> + </td> + </tr> + </tbody> </table> @@ -4529,7 +4562,18 @@ STRING.sha1() </td> </tr> - <tr> + <tr> + <td> + {% highlight scala %} +STRING.sha224() +{% endhighlight %} + </td> + <td> + <p>Returns the SHA-224 hash of the string argument as a string of 56 hexadecimal digits; null if <i>string</i> is null.</p> + </td> + </tr> + + <tr> <td> {% highlight scala %} STRING.sha256() @@ -4539,7 +4583,28 @@ STRING.sha256() <p>Returns the SHA-256 hash of the string argument as a string of 64 hexadecimal digits; null if <i>string</i> is null.</p> </td> </tr> + + <tr> + <td> + {% highlight scala %} +STRING.sha384() +{% endhighlight %} + </td> + <td> + <p>Returns the SHA-384 hash of the string argument as a string of 96 hexadecimal digits; null if <i>string</i> is null.</p> + </td> + </tr> + <tr> + <td> + {% highlight scala %} +STRING.sha512() +{% endhighlight %} + </td> + <td> + <p>Returns the SHA-512 hash of the string argument as a string of 128 hexadecimal digits; null if <i>string</i> is null.</p> + </td> + </tr> </tbody> </table> http://git-wip-us.apache.org/repos/asf/flink/blob/5544ab5d/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/scala/expressionDsl.scala ---------------------------------------------------------------------- diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/scala/expressionDsl.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/scala/expressionDsl.scala index 09e7ef9..1494de6 100644 --- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/scala/expressionDsl.scala +++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/scala/expressionDsl.scala @@ -773,11 +773,32 @@ trait ImplicitExpressionOperations { def sha1() = Sha1(expr) /** + * Returns the SHA-224 hash of the string argument; null if string is null. + * + * @return string of 64 hexadecimal digits or null + */ + def sha224() = Sha224(expr) + + /** * Returns the SHA-256 hash of the string argument; null if string is null. * * @return string of 64 hexadecimal digits or null */ def sha256() = Sha256(expr) + + /** + * Returns the SHA-384 hash of the string argument; null if string is null. + * + * @return string of 64 hexadecimal digits or null + */ + def sha384() = Sha384(expr) + + /** + * Returns the SHA-512 hash of the string argument; null if string is null. + * + * @return string of 64 hexadecimal digits or null + */ + def sha512() = Sha512(expr) } /** http://git-wip-us.apache.org/repos/asf/flink/blob/5544ab5d/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/calls/FunctionGenerator.scala ---------------------------------------------------------------------- diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/calls/FunctionGenerator.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/calls/FunctionGenerator.scala index 2cd7388..0f2e986 100644 --- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/calls/FunctionGenerator.scala +++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/calls/FunctionGenerator.scala @@ -560,11 +560,29 @@ object FunctionGenerator { ) addSqlFunction( + ScalarSqlFunctions.SHA224, + Seq(STRING_TYPE_INFO), + new HashCalcCallGen("SHA-224") + ) + + addSqlFunction( ScalarSqlFunctions.SHA256, Seq(STRING_TYPE_INFO), new HashCalcCallGen("SHA-256") ) + addSqlFunction( + ScalarSqlFunctions.SHA384, + Seq(STRING_TYPE_INFO), + new HashCalcCallGen("SHA-384") + ) + + addSqlFunction( + ScalarSqlFunctions.SHA512, + Seq(STRING_TYPE_INFO), + new HashCalcCallGen("SHA-512") + ) + // ---------------------------------------------------------------------------------------------- /** http://git-wip-us.apache.org/repos/asf/flink/blob/5544ab5d/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/expressions/hashExpressions.scala ---------------------------------------------------------------------- diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/expressions/hashExpressions.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/expressions/hashExpressions.scala index 2880ad4..59a3635 100644 --- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/expressions/hashExpressions.scala +++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/expressions/hashExpressions.scala @@ -50,6 +50,19 @@ case class Sha1(child: Expression) extends UnaryExpression with InputTypeSpec { } } +case class Sha224(child: Expression) extends UnaryExpression with InputTypeSpec { + + override private[flink] def resultType: TypeInformation[_] = STRING_TYPE_INFO + + override private[flink] def expectedTypes: Seq[TypeInformation[_]] = STRING_TYPE_INFO :: Nil + + override def toString: String = s"($child).sha224()" + + override private[flink] def toRexNode(implicit relBuilder: RelBuilder): RexNode = { + relBuilder.call(ScalarSqlFunctions.SHA224, child.toRexNode) + } +} + case class Sha256(child: Expression) extends UnaryExpression with InputTypeSpec { override private[flink] def resultType: TypeInformation[_] = STRING_TYPE_INFO @@ -62,3 +75,29 @@ case class Sha256(child: Expression) extends UnaryExpression with InputTypeSpec relBuilder.call(ScalarSqlFunctions.SHA256, child.toRexNode) } } + +case class Sha384(child: Expression) extends UnaryExpression with InputTypeSpec { + + override private[flink] def resultType: TypeInformation[_] = STRING_TYPE_INFO + + override private[flink] def expectedTypes: Seq[TypeInformation[_]] = STRING_TYPE_INFO :: Nil + + override def toString: String = s"($child).sha384()" + + override private[flink] def toRexNode(implicit relBuilder: RelBuilder): RexNode = { + relBuilder.call(ScalarSqlFunctions.SHA384, child.toRexNode) + } +} + +case class Sha512(child: Expression) extends UnaryExpression with InputTypeSpec { + + override private[flink] def resultType: TypeInformation[_] = STRING_TYPE_INFO + + override private[flink] def expectedTypes: Seq[TypeInformation[_]] = STRING_TYPE_INFO :: Nil + + override def toString: String = s"($child).sha512()" + + override private[flink] def toRexNode(implicit relBuilder: RelBuilder): RexNode = { + relBuilder.call(ScalarSqlFunctions.SHA512, child.toRexNode) + } +} http://git-wip-us.apache.org/repos/asf/flink/blob/5544ab5d/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/functions/sql/ScalarSqlFunctions.scala ---------------------------------------------------------------------- diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/functions/sql/ScalarSqlFunctions.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/functions/sql/ScalarSqlFunctions.scala index 891aba9..b0c6d44 100644 --- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/functions/sql/ScalarSqlFunctions.scala +++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/functions/sql/ScalarSqlFunctions.scala @@ -103,6 +103,15 @@ object ScalarSqlFunctions { SqlFunctionCategory.STRING ) + val SHA224 = new SqlFunction( + "SHA224", + SqlKind.OTHER_FUNCTION, + ReturnTypes.ARG0_NULLABLE, + InferTypes.RETURN_TYPE, + OperandTypes.STRING, + SqlFunctionCategory.STRING + ) + val SHA256 = new SqlFunction( "SHA256", SqlKind.OTHER_FUNCTION, @@ -112,6 +121,24 @@ object ScalarSqlFunctions { SqlFunctionCategory.STRING ) + val SHA384 = new SqlFunction( + "SHA384", + SqlKind.OTHER_FUNCTION, + ReturnTypes.ARG0_NULLABLE, + InferTypes.RETURN_TYPE, + OperandTypes.STRING, + SqlFunctionCategory.STRING + ) + + val SHA512 = new SqlFunction( + "SHA512", + SqlKind.OTHER_FUNCTION, + ReturnTypes.ARG0_NULLABLE, + InferTypes.RETURN_TYPE, + OperandTypes.STRING, + SqlFunctionCategory.STRING + ) + val DATE_FORMAT = new SqlFunction( "DATE_FORMAT", SqlKind.OTHER_FUNCTION, http://git-wip-us.apache.org/repos/asf/flink/blob/5544ab5d/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/validate/FunctionCatalog.scala ---------------------------------------------------------------------- diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/validate/FunctionCatalog.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/validate/FunctionCatalog.scala index 51aac27..56e6cd6 100644 --- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/validate/FunctionCatalog.scala +++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/validate/FunctionCatalog.scala @@ -273,7 +273,10 @@ object FunctionCatalog { // crypto hash "md5" -> classOf[Md5], "sha1" -> classOf[Sha1], - "sha256" -> classOf[Sha256] + "sha224" -> classOf[Sha224], + "sha256" -> classOf[Sha256], + "sha384" -> classOf[Sha384], + "sha512" -> classOf[Sha512] ) /** @@ -432,8 +435,10 @@ class BasicOperatorTable extends ReflectiveSqlOperatorTable { ScalarSqlFunctions.RPAD, ScalarSqlFunctions.MD5, ScalarSqlFunctions.SHA1, + ScalarSqlFunctions.SHA224, ScalarSqlFunctions.SHA256, - + ScalarSqlFunctions.SHA384, + ScalarSqlFunctions.SHA512, // EXTENSIONS BasicOperatorTable.TUMBLE, BasicOperatorTable.HOP, http://git-wip-us.apache.org/repos/asf/flink/blob/5544ab5d/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/expressions/ScalarFunctionsTest.scala ---------------------------------------------------------------------- diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/expressions/ScalarFunctionsTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/expressions/ScalarFunctionsTest.scala index a9d2733..8a2bc33 100644 --- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/expressions/ScalarFunctionsTest.scala +++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/expressions/ScalarFunctionsTest.scala @@ -1843,7 +1843,10 @@ class ScalarFunctionsTest extends ScalarTypesTestBase { def testHashFunctions(): Unit = { val expectedMd5 = "098f6bcd4621d373cade4e832627b4f6" val expectedSha1 = "a94a8fe5ccb19ba61c4c0873d391e987982fbbd3" + val expectedSha224 = "90a3ed9e32b2aaf4c61c410eb925426119e1a9dc53d4286ade99a809" val expectedSha256 = "9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08" + val expectedSha384 = "768412320f7b0aa5812fce428dc4706b3cae50e02a64caa16a782249bfe8efc4b7ef1ccb126255d196047dfedf17a0a9" // scalastyle:ignore + val expectedSha512 = "ee26b0dd4af7e749aa1a8ee3c10ae9923f618980772e473f8819a5d4940e0db27ac185f8a0e1d5f84f88bc887fd67b143732c304cc5fa9ad8e6f57f50028a8ff" // scalastyle:ignore testAllApis( "test".md5(), @@ -1858,21 +1861,45 @@ class ScalarFunctionsTest extends ScalarTypesTestBase { expectedSha1) testAllApis( + "test".sha224(), + "sha224('test')", + "SHA224('test')", + expectedSha224) + + testAllApis( "test".sha256(), "sha256('test')", "SHA256('test')", expectedSha256) testAllApis( + "test".sha384(), + "sha384('test')", + "SHA384('test')", + expectedSha384) + + testAllApis( + "test".sha512(), + "sha512('test')", + "SHA512('test')", + expectedSha512) + + testAllApis( 'f33.md5(), - "sha256(f33)", - "SHA256(f33)", + "md5(f33)", + "MD5(f33)", "null") testAllApis( - 'f33.sha256(), - "sha256(f33)", - "SHA256(f33)", + 'f33.sha1(), + "sha1(f33)", + "SHA1(f33)", + "null") + + testAllApis( + 'f33.sha224(), + "sha224(f33)", + "SHA224(f33)", "null") testAllApis( @@ -1880,6 +1907,18 @@ class ScalarFunctionsTest extends ScalarTypesTestBase { "sha256(f33)", "SHA256(f33)", "null") + + testAllApis( + 'f33.sha384(), + "sha384(f33)", + "SHA384(f33)", + "null") + + testAllApis( + 'f33.sha512(), + "sha512(f33)", + "SHA512(f33)", + "null") } // ---------------------------------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/flink/blob/5544ab5d/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/expressions/SqlExpressionTest.scala ---------------------------------------------------------------------- diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/expressions/SqlExpressionTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/expressions/SqlExpressionTest.scala index cf4b583..81ac02e 100644 --- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/expressions/SqlExpressionTest.scala +++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/expressions/SqlExpressionTest.scala @@ -145,12 +145,25 @@ class SqlExpressionTest extends ExpressionTestBase { testSqlApi("SHA1('')", "da39a3ee5e6b4b0d3255bfef95601890afd80709") testSqlApi("SHA1('test')", "a94a8fe5ccb19ba61c4c0873d391e987982fbbd3") + testSqlApi("SHA224('')", "d14a028c2a3a2bc9476102bb288234c415a2b01f828ea62ac5b3e42f") + testSqlApi("SHA224('test')", "90a3ed9e32b2aaf4c61c410eb925426119e1a9dc53d4286ade99a809") + testSqlApi("SHA256('')", "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855") testSqlApi("SHA256('test')", "9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08") + testSqlApi("SHA384('')", "38b060a751ac96384cd9327eb1b1e36a21fdb71114be07434c0cc7bf63f6e1da274edebfe76f65fbd51ad2f14898b95b") // scalastyle:ignore + testSqlApi("SHA384('test')", "768412320f7b0aa5812fce428dc4706b3cae50e02a64caa16a782249bfe8efc4b7ef1ccb126255d196047dfedf17a0a9") // scalastyle:ignore + + testSqlApi("SHA512('')", "cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce47d0d13c5d85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e") // scalastyle:ignore + testSqlApi("SHA512('test')", "ee26b0dd4af7e749aa1a8ee3c10ae9923f618980772e473f8819a5d4940e0db27ac185f8a0e1d5f84f88bc887fd67b143732c304cc5fa9ad8e6f57f50028a8ff") // scalastyle:ignore + + testSqlApi("MD5(CAST(NULL AS VARCHAR))", "null") testSqlApi("SHA1(CAST(NULL AS VARCHAR))", "null") + testSqlApi("SHA224(CAST(NULL AS VARCHAR))", "null") testSqlApi("SHA256(CAST(NULL AS VARCHAR))", "null") + testSqlApi("SHA384(CAST(NULL AS VARCHAR))", "null") + testSqlApi("SHA512(CAST(NULL AS VARCHAR))", "null") } @Test
