anishshri-db commented on code in PR #49121:
URL: https://github.com/apache/spark/pull/49121#discussion_r1883383031
##########
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/UnsupportedOperationChecker.scala:
##########
@@ -140,6 +142,37 @@ object UnsupportedOperationChecker extends Logging {
}
}
+ private def checkAvroSupportForStatefulOperator(p: LogicalPlan): Boolean = p
match {
+ // TODO: remove operators from this list as support for avro encoding is
added
+ case s: Aggregate if s.isStreaming => false
+ // Since the Distinct node will be replaced to Aggregate in the optimizer
rule
+ // [[ReplaceDistinctWithAggregate]], here we also need to check all
Distinct node by
+ // assuming it as Aggregate.
+ case d @ Distinct(_: LogicalPlan) if d.isStreaming => false
+ case _ @ Join(left, right, _, _, _) if left.isStreaming &&
right.isStreaming => false
+ case f: FlatMapGroupsWithState if f.isStreaming => false
+ case f: FlatMapGroupsInPandasWithState if f.isStreaming => false
+ case d: Deduplicate if d.isStreaming => false
+ case d: DeduplicateWithinWatermark if d.isStreaming => false
+ case _ => true
+ }
+
+ // Rule to check that avro encoding format is not supported in case any
+ // non-transformWithState stateful streaming operators are present in the
query.
+ def checkSupportedStoreEncodingFormats(plan: LogicalPlan): Unit = {
+ val storeEncodingFormat = SQLConf.get.stateStoreEncodingFormat
+ if (storeEncodingFormat.toLowerCase(Locale.ROOT) == "avro") {
+ plan.foreach { subPlan =>
+ if (!checkAvroSupportForStatefulOperator(subPlan)) {
+ val errorMsg = "State store encoding format as avro is not supported
for " +
Review Comment:
Done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
