[GitHub] [spark] beliefer commented on a diff in pull request #37169: [SPARK-38901][SQL] DS V2 supports push down misc functions

Mon, 25 Jul 2022 02:41:35 -0700 


beliefer commented on code in PR #37169:
URL: https://github.com/apache/spark/pull/37169#discussion_r928671144


##########
sql/core/src/main/scala/org/apache/spark/sql/catalyst/util/V2ExpressionBuilder.scala:
##########
@@ -397,6 +397,33 @@ class V2ExpressionBuilder(e: Expression, isPredicate: 
Boolean = false) {
       generateExpression(child).map(v => new V2Extract("WEEK", v))
     case YearOfWeek(child) =>
       generateExpression(child).map(v => new V2Extract("YEAR_OF_WEEK", v))
+    case encrypt: AesEncrypt =>
+      val childrenExpressions = encrypt.children.flatMap(generateExpression(_))
+      if (childrenExpressions.length == encrypt.children.length) {
+        Some(new GeneralScalarExpression("AES_ENCRYPT", 
childrenExpressions.toArray[V2Expression]))
+      } else {
+        None
+      }
+    case decrypt: AesDecrypt =>
+      val childrenExpressions = decrypt.children.flatMap(generateExpression(_))
+      if (childrenExpressions.length == decrypt.children.length) {
+        Some(new GeneralScalarExpression("AES_DECRYPT", 
childrenExpressions.toArray[V2Expression]))
+      } else {
+        None
+      }
+    case Crc32(child) => generateExpression(child)
+      .map(v => new GeneralScalarExpression("CRC32", Array[V2Expression](v)))

Review Comment:
   ditto



##########
sql/core/src/main/scala/org/apache/spark/sql/catalyst/util/V2ExpressionBuilder.scala:
##########
@@ -397,6 +397,33 @@ class V2ExpressionBuilder(e: Expression, isPredicate: 
Boolean = false) {
       generateExpression(child).map(v => new V2Extract("WEEK", v))
     case YearOfWeek(child) =>
       generateExpression(child).map(v => new V2Extract("YEAR_OF_WEEK", v))
+    case encrypt: AesEncrypt =>
+      val childrenExpressions = encrypt.children.flatMap(generateExpression(_))
+      if (childrenExpressions.length == encrypt.children.length) {
+        Some(new GeneralScalarExpression("AES_ENCRYPT", 
childrenExpressions.toArray[V2Expression]))
+      } else {
+        None
+      }

Review Comment:
   https://github.com/apache/spark/pull/37249 extract the common 
`generateExpressionWithName`, you can use it now.



##########
sql/core/src/main/scala/org/apache/spark/sql/catalyst/util/V2ExpressionBuilder.scala:
##########
@@ -397,6 +397,33 @@ class V2ExpressionBuilder(e: Expression, isPredicate: 
Boolean = false) {
       generateExpression(child).map(v => new V2Extract("WEEK", v))
     case YearOfWeek(child) =>
       generateExpression(child).map(v => new V2Extract("YEAR_OF_WEEK", v))
+    case encrypt: AesEncrypt =>
+      val childrenExpressions = encrypt.children.flatMap(generateExpression(_))
+      if (childrenExpressions.length == encrypt.children.length) {
+        Some(new GeneralScalarExpression("AES_ENCRYPT", 
childrenExpressions.toArray[V2Expression]))
+      } else {
+        None
+      }
+    case decrypt: AesDecrypt =>
+      val childrenExpressions = decrypt.children.flatMap(generateExpression(_))
+      if (childrenExpressions.length == decrypt.children.length) {
+        Some(new GeneralScalarExpression("AES_DECRYPT", 
childrenExpressions.toArray[V2Expression]))
+      } else {
+        None
+      }
+    case Crc32(child) => generateExpression(child)
+      .map(v => new GeneralScalarExpression("CRC32", Array[V2Expression](v)))
+    case Md5(child) => generateExpression(child)
+      .map(v => new GeneralScalarExpression("MD5", Array[V2Expression](v)))
+    case Sha1(child) => generateExpression(child)
+      .map(v => new GeneralScalarExpression("SHA1", Array[V2Expression](v)))

Review Comment:
   ditto



##########
sql/core/src/main/scala/org/apache/spark/sql/catalyst/util/V2ExpressionBuilder.scala:
##########
@@ -397,6 +397,33 @@ class V2ExpressionBuilder(e: Expression, isPredicate: 
Boolean = false) {
       generateExpression(child).map(v => new V2Extract("WEEK", v))
     case YearOfWeek(child) =>
       generateExpression(child).map(v => new V2Extract("YEAR_OF_WEEK", v))
+    case encrypt: AesEncrypt =>
+      val childrenExpressions = encrypt.children.flatMap(generateExpression(_))
+      if (childrenExpressions.length == encrypt.children.length) {
+        Some(new GeneralScalarExpression("AES_ENCRYPT", 
childrenExpressions.toArray[V2Expression]))
+      } else {
+        None
+      }
+    case decrypt: AesDecrypt =>
+      val childrenExpressions = decrypt.children.flatMap(generateExpression(_))
+      if (childrenExpressions.length == decrypt.children.length) {
+        Some(new GeneralScalarExpression("AES_DECRYPT", 
childrenExpressions.toArray[V2Expression]))
+      } else {
+        None
+      }
+    case Crc32(child) => generateExpression(child)
+      .map(v => new GeneralScalarExpression("CRC32", Array[V2Expression](v)))
+    case Md5(child) => generateExpression(child)
+      .map(v => new GeneralScalarExpression("MD5", Array[V2Expression](v)))

Review Comment:
   ditto



##########
sql/core/src/main/scala/org/apache/spark/sql/catalyst/util/V2ExpressionBuilder.scala:
##########
@@ -397,6 +397,33 @@ class V2ExpressionBuilder(e: Expression, isPredicate: 
Boolean = false) {
       generateExpression(child).map(v => new V2Extract("WEEK", v))
     case YearOfWeek(child) =>
       generateExpression(child).map(v => new V2Extract("YEAR_OF_WEEK", v))
+    case encrypt: AesEncrypt =>
+      val childrenExpressions = encrypt.children.flatMap(generateExpression(_))
+      if (childrenExpressions.length == encrypt.children.length) {
+        Some(new GeneralScalarExpression("AES_ENCRYPT", 
childrenExpressions.toArray[V2Expression]))
+      } else {
+        None
+      }
+    case decrypt: AesDecrypt =>
+      val childrenExpressions = decrypt.children.flatMap(generateExpression(_))
+      if (childrenExpressions.length == decrypt.children.length) {
+        Some(new GeneralScalarExpression("AES_DECRYPT", 
childrenExpressions.toArray[V2Expression]))
+      } else {
+        None
+      }
+    case Crc32(child) => generateExpression(child)
+      .map(v => new GeneralScalarExpression("CRC32", Array[V2Expression](v)))
+    case Md5(child) => generateExpression(child)
+      .map(v => new GeneralScalarExpression("MD5", Array[V2Expression](v)))
+    case Sha1(child) => generateExpression(child)
+      .map(v => new GeneralScalarExpression("SHA1", Array[V2Expression](v)))
+    case sha2: Sha2 =>
+      val childrenExpressions = sha2.children.flatMap(generateExpression(_))
+      if (childrenExpressions.length == sha2.children.length) {
+        Some(new GeneralScalarExpression("SHA2", 
childrenExpressions.toArray[V2Expression]))
+      } else {
+        None
+      }

Review Comment:
   ditto



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to