JingsongLi commented on a change in pull request #8127:
[FLINK-12140][table-planner-blink] Support e2e sort merge join operator in
batch mode
URL: https://github.com/apache/flink/pull/8127#discussion_r274286950
##########
File path:
flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/plan/nodes/physical/batch/BatchExecSortMergeJoin.scala
##########
@@ -104,6 +117,109 @@ trait BatchExecSortMergeJoinBase extends
BatchExecJoinBase {
val rowCount = mq.getRowCount(this)
costFactory.makeCost(rowCount, cpuCost, 0, 0, sortMemCost)
}
+
+ /**
+ * Now must be full dam without two input operator chain.
+ * TODO two input operator chain
+ */
+ override def getDamBehavior: DamBehavior = {
+ DamBehavior.FULL_DAM
+// if (!leftSorted && !rightSorted) {
+// DamBehavior.FULL_DAM
+// } else if (!leftSorted || !rightSorted) {
+// DamBehavior.MATERIALIZING
+// } else {
+// DamBehavior.PIPELINED
+// }
+ }
+
+ override def getInputNodes: util.List[ExecNode[BatchTableEnvironment, _]] =
+ getInputs.map(_.asInstanceOf[ExecNode[BatchTableEnvironment, _]])
+
+ override def translateToPlanInternal(
+ tableEnv: BatchTableEnvironment): StreamTransformation[BaseRow] = {
+
+ val config = tableEnv.getConfig
+
+ val leftInput = getInputNodes.get(0).translateToPlan(tableEnv)
+ .asInstanceOf[StreamTransformation[BaseRow]]
+ val rightInput = getInputNodes.get(1).translateToPlan(tableEnv)
+ .asInstanceOf[StreamTransformation[BaseRow]]
+
+ val leftType = TypeConverters.createInternalTypeFromTypeInfo(
+ leftInput.getOutputType).asInstanceOf[RowType]
+ val rightType = TypeConverters.createInternalTypeFromTypeInfo(
+ rightInput.getOutputType).asInstanceOf[RowType]
+
+ val keyType = new RowType(leftAllKey.map(leftType.getFieldTypes()(_)): _*)
+
+ val condFunc = generateCondition(config, leftType, rightType)
+
+ val externalBufferMemory = config.getConf.getInteger(
+ TableConfigOptions.SQL_RESOURCE_EXTERNAL_BUFFER_MEM) *
TableConfigOptions.SIZE_IN_MB
+
+ val sortMemory = config.getConf.getInteger(
+ TableConfigOptions.SQL_RESOURCE_SORT_BUFFER_MEM) *
TableConfigOptions.SIZE_IN_MB
+
+ def newSortGen(originalKeys: Array[Int], t: RowType): SortCodeGenerator = {
+ val originalOrders = originalKeys.map(_ => true)
+ val (keys, orders, nullsIsLast) = SortUtil.deduplicateSortKeys(
+ originalKeys,
+ originalOrders,
+ SortUtil.getNullDefaultOrders(originalOrders))
+ val types = keys.map(t.getTypeAt)
+ new SortCodeGenerator(config, keys, types, orders, nullsIsLast)
+ }
+
+ val leftSortGen = newSortGen(leftAllKey, leftType)
+ val rightSortGen = newSortGen(rightAllKey, rightType)
+
+ val operator = new SortMergeJoinOperator(
+ sortMemory,
+ sortMemory,
+ externalBufferMemory,
+ flinkJoinType match {
+ case FlinkJoinRelType.INNER =>
SortMergeJoinOperator.SortMergeJoinType.INNER
Review comment:
It doesn't seem very good. `FlinkJoinRelType` belongs to `table.plan`.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
