This is an automated email from the ASF dual-hosted git repository. github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/dev/pr-12564-b05195f4925e640fe5085bc0202b6c4cf58d504f in repository https://gitbox.apache.org/repos/asf/seatunnel.git
commit 3556e91cc9220f5e4c0537d960ff6c57d612e072 Author: chovygo <[email protected]> AuthorDate: Thu Oct 8 13:35:54 2026 +0000 [Feature] [Transform-V2] Add MD5 hash SQL function (#12564) Co-authored-by: zhaowei <[email protected]> --- docs/en/transforms/sql-functions.md | 11 ++++ docs/zh/transforms/sql-functions.md | 11 ++++ .../transform/sql/zeta/ZetaSQLFunction.java | 3 ++ .../seatunnel/transform/sql/zeta/ZetaSQLType.java | 1 + .../sql/zeta/functions/StringFunction.java | 27 ++++++++++ .../transform/sql/SQLHashFunctionsTest.java | 62 ++++++++++++++++++++++ .../sql/zeta/functions/StringFunctionTest.java | 7 +++ 7 files changed, 122 insertions(+) diff --git a/docs/en/transforms/sql-functions.md b/docs/en/transforms/sql-functions.md index a244f3bad0..44966f6820 100644 --- a/docs/en/transforms/sql-functions.md +++ b/docs/en/transforms/sql-functions.md @@ -351,6 +351,17 @@ Example: MURMUR64('hello world') MURMUR64(NAME) +### MD5 + +```MD5(string|binary) -> STRING``` + +Calculate the MD5 hash of the input string or binary value and return the result as a 32-character lowercase hexadecimal string, compatible with Hive's `md5` function. This method returns null if the input parameter is null. + +Example: + +MD5('hello world') +MD5(NAME) + ### SOUNDEX ```SOUNDEX(string) -> STRING``` diff --git a/docs/zh/transforms/sql-functions.md b/docs/zh/transforms/sql-functions.md index b7eac38fd1..ebdb15f6a8 100644 --- a/docs/zh/transforms/sql-functions.md +++ b/docs/zh/transforms/sql-functions.md @@ -352,6 +352,17 @@ select SPLIT(test,';') as arrays MURMUR64('hello world') MURMUR64(NAME) +### MD5 + +```MD5(string|binary) -> STRING``` + +计算输入字符串或二进制值的 MD5 哈希值，返回 32 字符的小写十六进制字符串，与 Hive 的 `md5` 函数兼容。如果输入参数为 null，则返回 null。 + +示例: + +MD5('hello world') +MD5(NAME) + ### SOUNDEX ```SOUNDEX(string) -> STRING``` diff --git a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sql/zeta/ZetaSQLFunction.java b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sql/zeta/ZetaSQLFunction.java index 76765466a3..ae10a9dd92 100644 --- a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sql/zeta/ZetaSQLFunction.java +++ b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sql/zeta/ZetaSQLFunction.java @@ -129,6 +129,7 @@ public class ZetaSQLFunction { public static final String TRANSLATE = "TRANSLATE"; public static final String SPLIT = "SPLIT"; public static final String MURMUR64 = "MURMUR64"; + public static final String MD5 = "MD5"; // -------------------------crypto functions---------------------------- public static final String AES_ENCRYPT = "AES_ENCRYPT"; @@ -537,6 +538,8 @@ public class ZetaSQLFunction { return StringFunction.split(args); case MURMUR64: return StringFunction.murmur64(args); + case MD5: + return StringFunction.md5(args); case AES_ENCRYPT: return CryptoFunction.aesEncrypt(args); case AES_DECRYPT: diff --git a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sql/zeta/ZetaSQLType.java b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sql/zeta/ZetaSQLType.java index 8082d3b303..b536a0746d 100644 --- a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sql/zeta/ZetaSQLType.java +++ b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sql/zeta/ZetaSQLType.java @@ -327,6 +327,7 @@ public class ZetaSQLType { case ZetaSQLFunction.CONCAT_WS: case ZetaSQLFunction.HEXTORAW: case ZetaSQLFunction.RAWTOHEX: + case ZetaSQLFunction.MD5: case ZetaSQLFunction.TO_BASE64: case ZetaSQLFunction.FROM_BASE64: case ZetaSQLFunction.INSERT: diff --git a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sql/zeta/functions/StringFunction.java b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sql/zeta/functions/StringFunction.java index cb1375cefc..ace22df5de 100644 --- a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sql/zeta/functions/StringFunction.java +++ b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sql/zeta/functions/StringFunction.java @@ -836,4 +836,31 @@ public class StringFunction { } return Hashing.murmur3_128().hashString(arg, StandardCharsets.UTF_8).asLong(); } + + /** + * Computes the MD5 digest of the input and returns it as a 32-character lowercase hexadecimal + * string, matching Hive's {@code md5(string|binary)}. + * + * <p>String inputs are hashed as UTF-8 bytes; binary (byte[]) inputs are hashed as raw bytes. + * Non-string, non-binary values are converted via {@code toString()} and hashed as UTF-8, + * mirroring Hive's implicit cast to string. + * + * @param args list containing a single string, binary, or implicitly castable value + * @return 32-char lowercase hex MD5 digest, or null if the input is null + */ + @SuppressWarnings( + "deprecation") // Guava deprecates MD5 (broken for security); Hive parity requires it + public static String md5(List<Object> args) { + if (args.size() != 1) { + throw new IllegalArgumentException("MD5 expects exactly 1 argument"); + } + Object arg = args.get(0); + if (arg == null) { + return null; + } + if (arg instanceof byte[]) { + return Hashing.md5().hashBytes((byte[]) arg).toString(); + } + return Hashing.md5().hashString(arg.toString(), StandardCharsets.UTF_8).toString(); + } } diff --git a/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/sql/SQLHashFunctionsTest.java b/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/sql/SQLHashFunctionsTest.java index df1572f35f..8301af0f87 100644 --- a/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/sql/SQLHashFunctionsTest.java +++ b/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/sql/SQLHashFunctionsTest.java @@ -107,4 +107,66 @@ public class SQLHashFunctionsTest { Assertions.assertEquals(outRow1.getField(0), outRow2.getField(0)); Assertions.assertEquals(murmur64Direct("test123"), outRow1.getField(0)); } + + @Test + public void testMd5WithNormalString() { + SeaTunnelRowType rowType = + new SeaTunnelRowType( + new String[] {"text"}, new SeaTunnelDataType[] {BasicType.STRING_TYPE}); + + SeaTunnelRow outRow = runSql("select MD5(text) as hash from dual", rowType, "abc"); + + Assertions.assertInstanceOf(String.class, outRow.getField(0)); + Assertions.assertEquals("900150983cd24fb0d6963f7d28e17f72", outRow.getField(0)); + } + + @Test + public void testMd5WithEmptyString() { + SeaTunnelRowType rowType = + new SeaTunnelRowType( + new String[] {"text"}, new SeaTunnelDataType[] {BasicType.STRING_TYPE}); + + SeaTunnelRow outRow = runSql("select MD5(text) as hash from dual", rowType, ""); + + Assertions.assertEquals("d41d8cd98f00b204e9800998ecf8427e", outRow.getField(0)); + } + + @Test + public void testMd5WithNull() { + SeaTunnelRowType rowType = + new SeaTunnelRowType( + new String[] {"text"}, new SeaTunnelDataType[] {BasicType.STRING_TYPE}); + + SeaTunnelRow outRow = runSql("select MD5(text) as hash from dual", rowType, (Object) null); + + Assertions.assertNull(outRow.getField(0)); + } + + @Test + public void testMd5WithMultibyteUtf8() { + // Multi-byte UTF-8 path: Hive md5 hashes the UTF-8 bytes of the string. + SeaTunnelRowType rowType = + new SeaTunnelRowType( + new String[] {"text"}, new SeaTunnelDataType[] {BasicType.STRING_TYPE}); + + SeaTunnelRow outRow = runSql("select MD5(text) as hash from dual", rowType, "你好"); + + Assertions.assertEquals("7eca689f0d3389d9dea66ae112e5cfd7", outRow.getField(0)); + } + + @Test + public void testMd5HiveParity() { + // Hive canonical vector: MD5('The quick brown fox jumps over the lazy dog') + SeaTunnelRowType rowType = + new SeaTunnelRowType( + new String[] {"text"}, new SeaTunnelDataType[] {BasicType.STRING_TYPE}); + + SeaTunnelRow outRow = + runSql( + "select MD5(text) as hash from dual", + rowType, + "The quick brown fox jumps over the lazy dog"); + + Assertions.assertEquals("9e107d9d372bb6826bd81d3542a419d6", outRow.getField(0)); + } } diff --git a/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/sql/zeta/functions/StringFunctionTest.java b/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/sql/zeta/functions/StringFunctionTest.java index d226c49b46..21a713d4d8 100644 --- a/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/sql/zeta/functions/StringFunctionTest.java +++ b/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/sql/zeta/functions/StringFunctionTest.java @@ -320,6 +320,13 @@ public class StringFunctionTest { Assertions.assertEquals("U2VhVHVubmVs", StringFunction.toBase64(args)); } + @Test + public void testMd5WithBytesInput() { + List<Object> args = new ArrayList<>(); + args.add("abc".getBytes(StandardCharsets.UTF_8)); + Assertions.assertEquals("900150983cd24fb0d6963f7d28e17f72", StringFunction.md5(args)); + } + @Test public void testToBase64BytesInputRejectsCharset() { IllegalArgumentException bytesWithCharset =
