This is an automated email from the ASF dual-hosted git repository. twalthr pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/flink.git.
from 330aae0 [FLINK-26847][python] Ensure command line option '-py' works in YARN application mode add b0b92cb [hotfix][core] Add method to get all ConfigOptions from an "Options" class add b8cdb8d [hotfix][table-planner] Remove consumed options not affecting the topology. add bc11450 [FLINK-26075][table-planner] Persist node configuration to JSON plan new 84723ee [FLINK-26075][table-planner][test] Regenerate JSON plans The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../apache/flink/configuration/ConfigUtils.java | 23 ++ .../table/planner/plan/nodes/exec/ExecNode.java | 1 + .../planner/plan/nodes/exec/ExecNodeBase.java | 21 +- .../planner/plan/nodes/exec/ExecNodeConfig.java | 8 +- .../planner/plan/nodes/exec/ExecNodeContext.java | 30 ++- .../exec/batch/BatchExecBoundedStreamScan.java | 3 + .../plan/nodes/exec/batch/BatchExecCalc.java | 3 + .../plan/nodes/exec/batch/BatchExecCorrelate.java | 3 + .../plan/nodes/exec/batch/BatchExecExchange.java | 8 +- .../plan/nodes/exec/batch/BatchExecExpand.java | 3 + .../nodes/exec/batch/BatchExecHashAggregate.java | 3 + .../plan/nodes/exec/batch/BatchExecHashJoin.java | 3 + .../exec/batch/BatchExecHashWindowAggregate.java | 3 + .../plan/nodes/exec/batch/BatchExecLegacySink.java | 3 + .../exec/batch/BatchExecLegacyTableSourceScan.java | 4 + .../plan/nodes/exec/batch/BatchExecLimit.java | 3 + .../plan/nodes/exec/batch/BatchExecLookupJoin.java | 3 + .../nodes/exec/batch/BatchExecMultipleInput.java | 3 + .../nodes/exec/batch/BatchExecNestedLoopJoin.java | 3 + .../nodes/exec/batch/BatchExecOverAggregate.java | 3 + .../exec/batch/BatchExecOverAggregateBase.java | 10 +- .../plan/nodes/exec/batch/BatchExecPythonCalc.java | 3 + .../nodes/exec/batch/BatchExecPythonCorrelate.java | 3 + .../exec/batch/BatchExecPythonGroupAggregate.java | 4 + .../batch/BatchExecPythonGroupWindowAggregate.java | 4 + .../exec/batch/BatchExecPythonOverAggregate.java | 3 + .../plan/nodes/exec/batch/BatchExecRank.java | 3 + .../plan/nodes/exec/batch/BatchExecSink.java | 3 + .../plan/nodes/exec/batch/BatchExecSort.java | 3 + .../nodes/exec/batch/BatchExecSortAggregate.java | 3 + .../plan/nodes/exec/batch/BatchExecSortLimit.java | 3 + .../nodes/exec/batch/BatchExecSortMergeJoin.java | 3 + .../exec/batch/BatchExecSortWindowAggregate.java | 3 + .../nodes/exec/batch/BatchExecTableSourceScan.java | 7 +- .../plan/nodes/exec/batch/BatchExecUnion.java | 7 +- .../plan/nodes/exec/batch/BatchExecValues.java | 8 +- .../exec/batch/BatchExecWindowTableFunction.java | 3 + .../plan/nodes/exec/common/CommonExecCalc.java | 4 +- .../nodes/exec/common/CommonExecCorrelate.java | 4 +- .../plan/nodes/exec/common/CommonExecExchange.java | 4 +- .../plan/nodes/exec/common/CommonExecExpand.java | 4 +- .../nodes/exec/common/CommonExecLegacySink.java | 10 +- .../common/CommonExecLegacyTableSourceScan.java | 4 +- .../nodes/exec/common/CommonExecLookupJoin.java | 4 +- .../nodes/exec/common/CommonExecPythonCalc.java | 4 +- .../exec/common/CommonExecPythonCorrelate.java | 4 +- .../plan/nodes/exec/common/CommonExecSink.java | 3 +- .../exec/common/CommonExecTableSourceScan.java | 3 +- .../plan/nodes/exec/common/CommonExecUnion.java | 4 +- .../plan/nodes/exec/common/CommonExecValues.java | 4 +- .../exec/common/CommonExecWindowTableFunction.java | 4 +- .../processor/ForwardHashExchangeProcessor.java | 32 ++- .../MultipleInputNodeCreationProcessor.java | 38 ++- .../utils/InputPriorityConflictResolver.java | 19 +- ...zer.java => ConfigurationJsonDeserializer.java} | 27 ++- ...lizer.java => ConfigurationJsonSerializer.java} | 21 +- .../serde/ConfigurationJsonSerializerFilter.java | 28 ++- .../plan/nodes/exec/serde/JsonSerdeUtil.java | 3 + .../nodes/exec/stream/StreamExecAggregateBase.java | 4 +- .../plan/nodes/exec/stream/StreamExecCalc.java | 5 + .../exec/stream/StreamExecChangelogNormalize.java | 7 +- .../nodes/exec/stream/StreamExecCorrelate.java | 5 + .../exec/stream/StreamExecDataStreamScan.java | 3 + .../nodes/exec/stream/StreamExecDeduplicate.java | 6 +- .../exec/stream/StreamExecDropUpdateBefore.java | 10 +- .../plan/nodes/exec/stream/StreamExecExchange.java | 11 +- .../plan/nodes/exec/stream/StreamExecExpand.java | 5 + .../stream/StreamExecGlobalGroupAggregate.java | 13 +- .../stream/StreamExecGlobalWindowAggregate.java | 7 +- .../exec/stream/StreamExecGroupAggregate.java | 12 +- .../exec/stream/StreamExecGroupTableAggregate.java | 4 + .../stream/StreamExecGroupWindowAggregate.java | 7 +- .../StreamExecIncrementalGroupAggregate.java | 13 +- .../nodes/exec/stream/StreamExecIntervalJoin.java | 5 +- .../plan/nodes/exec/stream/StreamExecJoin.java | 6 +- .../nodes/exec/stream/StreamExecLegacySink.java | 3 + .../stream/StreamExecLegacyTableSourceScan.java | 4 + .../plan/nodes/exec/stream/StreamExecLimit.java | 7 +- .../exec/stream/StreamExecLocalGroupAggregate.java | 7 +- .../stream/StreamExecLocalWindowAggregate.java | 7 +- .../nodes/exec/stream/StreamExecLookupJoin.java | 5 + .../plan/nodes/exec/stream/StreamExecMatch.java | 5 +- .../exec/stream/StreamExecMiniBatchAssigner.java | 6 +- .../nodes/exec/stream/StreamExecMultipleInput.java | 7 +- .../nodes/exec/stream/StreamExecOverAggregate.java | 7 +- .../nodes/exec/stream/StreamExecPythonCalc.java | 6 +- .../exec/stream/StreamExecPythonCorrelate.java | 5 + .../stream/StreamExecPythonGroupAggregate.java | 7 +- .../StreamExecPythonGroupTableAggregate.java | 4 + .../StreamExecPythonGroupWindowAggregate.java | 10 +- .../exec/stream/StreamExecPythonOverAggregate.java | 10 +- .../plan/nodes/exec/stream/StreamExecRank.java | 8 +- .../plan/nodes/exec/stream/StreamExecSink.java | 6 +- .../plan/nodes/exec/stream/StreamExecSort.java | 3 + .../nodes/exec/stream/StreamExecSortLimit.java | 7 +- .../exec/stream/StreamExecTableSourceScan.java | 10 +- .../nodes/exec/stream/StreamExecTemporalJoin.java | 7 +- .../nodes/exec/stream/StreamExecTemporalSort.java | 6 +- .../plan/nodes/exec/stream/StreamExecUnion.java | 10 +- .../plan/nodes/exec/stream/StreamExecValues.java | 11 +- .../exec/stream/StreamExecWatermarkAssigner.java | 7 +- .../exec/stream/StreamExecWindowAggregate.java | 6 +- .../exec/stream/StreamExecWindowAggregateBase.java | 4 +- .../exec/stream/StreamExecWindowDeduplicate.java | 6 +- .../nodes/exec/stream/StreamExecWindowJoin.java | 6 +- .../nodes/exec/stream/StreamExecWindowRank.java | 6 +- .../exec/stream/StreamExecWindowTableFunction.java | 14 +- .../planner/plan/utils/ExecNodeMetadataUtil.java | 79 +++++++ .../batch/BatchPhysicalBoundedStreamScan.scala | 2 + .../nodes/physical/batch/BatchPhysicalCalc.scala | 4 +- .../physical/batch/BatchPhysicalCorrelate.scala | 4 +- .../physical/batch/BatchPhysicalExchange.scala | 1 + .../nodes/physical/batch/BatchPhysicalExpand.scala | 7 +- .../batch/BatchPhysicalHashAggregate.scala | 4 +- .../physical/batch/BatchPhysicalHashJoin.scala | 25 +- .../batch/BatchPhysicalHashWindowAggregate.scala | 5 +- .../physical/batch/BatchPhysicalLegacySink.scala | 5 +- .../batch/BatchPhysicalLegacyTableSourceScan.scala | 2 + .../nodes/physical/batch/BatchPhysicalLimit.scala | 16 +- .../batch/BatchPhysicalLocalHashAggregate.scala | 7 +- .../BatchPhysicalLocalHashWindowAggregate.scala | 5 +- .../batch/BatchPhysicalLocalSortAggregate.scala | 7 +- .../BatchPhysicalLocalSortWindowAggregate.scala | 5 +- .../physical/batch/BatchPhysicalLookupJoin.scala | 2 + .../batch/BatchPhysicalNestedLoopJoin.scala | 5 +- .../batch/BatchPhysicalOverAggregate.scala | 5 +- .../physical/batch/BatchPhysicalPythonCalc.scala | 2 + .../batch/BatchPhysicalPythonCorrelate.scala | 7 +- .../batch/BatchPhysicalPythonGroupAggregate.scala | 8 +- .../BatchPhysicalPythonGroupWindowAggregate.scala | 5 +- .../batch/BatchPhysicalPythonOverAggregate.scala | 5 +- .../nodes/physical/batch/BatchPhysicalRank.scala | 4 +- .../nodes/physical/batch/BatchPhysicalSink.scala | 8 +- .../nodes/physical/batch/BatchPhysicalSort.scala | 7 +- .../batch/BatchPhysicalSortAggregate.scala | 4 +- .../physical/batch/BatchPhysicalSortLimit.scala | 7 +- .../batch/BatchPhysicalSortMergeJoin.scala | 5 +- .../batch/BatchPhysicalSortWindowAggregate.scala | 5 +- .../batch/BatchPhysicalTableSourceScan.scala | 2 + .../nodes/physical/batch/BatchPhysicalUnion.scala | 7 +- .../nodes/physical/batch/BatchPhysicalValues.scala | 5 +- .../batch/BatchPhysicalWindowTableFunction.scala | 5 +- .../nodes/physical/stream/StreamPhysicalCalc.scala | 2 + .../stream/StreamPhysicalChangelogNormalize.scala | 2 + .../physical/stream/StreamPhysicalCorrelate.scala | 4 +- .../stream/StreamPhysicalDataStreamScan.scala | 2 + .../stream/StreamPhysicalDeduplicate.scala | 7 +- .../stream/StreamPhysicalDropUpdateBefore.scala | 7 +- .../physical/stream/StreamPhysicalExchange.scala | 4 +- .../physical/stream/StreamPhysicalExpand.scala | 7 +- .../StreamPhysicalGlobalGroupAggregate.scala | 7 +- .../StreamPhysicalGlobalWindowAggregate.scala | 4 +- .../stream/StreamPhysicalGroupAggregate.scala | 7 +- .../stream/StreamPhysicalGroupTableAggregate.scala | 7 +- .../StreamPhysicalGroupWindowAggregate.scala | 5 +- .../StreamPhysicalGroupWindowTableAggregate.scala | 5 +- .../StreamPhysicalIncrementalGroupAggregate.scala | 7 +- .../stream/StreamPhysicalIntervalJoin.scala | 12 +- .../nodes/physical/stream/StreamPhysicalJoin.scala | 18 +- .../physical/stream/StreamPhysicalLegacySink.scala | 5 +- .../StreamPhysicalLegacyTableSourceScan.scala | 2 + .../physical/stream/StreamPhysicalLimit.scala | 19 +- .../stream/StreamPhysicalLocalGroupAggregate.scala | 7 +- .../StreamPhysicalLocalWindowAggregate.scala | 4 +- .../physical/stream/StreamPhysicalLookupJoin.scala | 2 + .../physical/stream/StreamPhysicalMatch.scala | 7 +- .../stream/StreamPhysicalMiniBatchAssigner.scala | 7 +- .../stream/StreamPhysicalOverAggregate.scala | 7 +- .../physical/stream/StreamPhysicalPythonCalc.scala | 4 +- .../stream/StreamPhysicalPythonCorrelate.scala | 7 +- .../StreamPhysicalPythonGroupAggregate.scala | 7 +- .../StreamPhysicalPythonGroupTableAggregate.scala | 7 +- .../StreamPhysicalPythonGroupWindowAggregate.scala | 5 +- .../stream/StreamPhysicalPythonOverAggregate.scala | 7 +- .../nodes/physical/stream/StreamPhysicalRank.scala | 5 +- .../nodes/physical/stream/StreamPhysicalSink.scala | 5 +- .../nodes/physical/stream/StreamPhysicalSort.scala | 5 +- .../physical/stream/StreamPhysicalSortLimit.scala | 7 +- .../stream/StreamPhysicalTableSourceScan.scala | 2 + .../stream/StreamPhysicalTemporalJoin.scala | 2 + .../stream/StreamPhysicalTemporalSort.scala | 5 +- .../physical/stream/StreamPhysicalUnion.scala | 7 +- .../physical/stream/StreamPhysicalValues.scala | 5 +- .../stream/StreamPhysicalWatermarkAssigner.scala | 2 + .../stream/StreamPhysicalWindowAggregate.scala | 4 +- .../stream/StreamPhysicalWindowDeduplicate.scala | 5 +- .../physical/stream/StreamPhysicalWindowJoin.scala | 2 + .../physical/stream/StreamPhysicalWindowRank.scala | 5 +- .../stream/StreamPhysicalWindowTableFunction.scala | 5 +- .../apache/flink/table/api/CompiledPlanITCase.java | 51 +++- .../utils/InputPriorityConflictResolverTest.java | 1 + .../plan/utils/ExecNodeMetadataUtilTest.java | 256 ++++++++++++++++++--- .../test/resources/jsonplan/testGetJsonPlan.out | 6 + .../CalcJsonPlanTest_jsonplan/testComplexCalc.out | 6 + .../CalcJsonPlanTest_jsonplan/testSimpleFilter.out | 6 + .../testSimpleProject.out | 6 + .../testChangelogSource.out | 10 + .../testUpsertSource.out | 10 + .../testCrossJoin.out | 6 + .../testCrossJoinOverrideParameters.out | 6 + .../testJoinWithFilter.out | 6 + .../testLeftOuterJoinWithLiteralTrue.out | 6 + .../testDeduplication.out | 12 + .../ExpandJsonPlanTest_jsonplan/testExpand.out | 14 ++ ...tDistinctAggCalls[isMiniBatchEnabled=false].out | 10 + ...stDistinctAggCalls[isMiniBatchEnabled=true].out | 14 ++ ...gCallsWithGroupBy[isMiniBatchEnabled=false].out | 10 + ...ggCallsWithGroupBy[isMiniBatchEnabled=true].out | 14 ++ ...AggWithoutGroupBy[isMiniBatchEnabled=false].out | 10 + ...eAggWithoutGroupBy[isMiniBatchEnabled=true].out | 14 ++ ...erDefinedAggCalls[isMiniBatchEnabled=false].out | 10 + ...serDefinedAggCalls[isMiniBatchEnabled=true].out | 10 + .../testEventTimeHopWindow.out | 11 + .../testEventTimeSessionWindow.out | 11 + .../testEventTimeTumbleWindow.out | 11 + .../testProcTimeHopWindow.out | 11 + .../testProcTimeSessionWindow.out | 11 + .../testProcTimeTumbleWindow.out | 11 + .../testIncrementalAggregate.out | 18 ++ ...lAggregateWithSumCountDistinctAndRetraction.out | 26 +++ .../testProcessingTimeInnerJoinWithOnClause.out | 6 + .../testRowTimeInnerJoinWithOnClause.out | 6 + .../JoinJsonPlanTest_jsonplan/testInnerJoin.out | 6 + .../testInnerJoinWithEqualPk.out | 14 ++ .../testInnerJoinWithPk.out | 14 ++ .../testLeftJoinNonEqui.out | 6 + .../LimitJsonPlanTest_jsonplan/testLimit.out | 9 + .../testJoinTemporalTable.out | 6 + ...testJoinTemporalTableWithProjectionPushDown.out | 6 + .../testMatch.out | 6 + .../testProcTimeBoundedNonPartitionedRangeOver.out | 6 + .../testProcTimeBoundedPartitionedRangeOver.out | 6 + ...undedPartitionedRowsOverWithBuiltinProctime.out | 6 + .../testProcTimeUnboundedPartitionedRangeOver.out | 6 + ...stProctimeBoundedDistinctPartitionedRowOver.out | 6 + ...edDistinctWithNonDistinctPartitionedRowOver.out | 6 + .../testRowTimeBoundedPartitionedRowsOver.out | 6 + .../stream/RankJsonPlanTest_jsonplan/testRank.out | 9 + .../testSortLimit.out | 9 + .../testOverwrite.out | 6 + .../testPartitioning.out | 6 + .../testWritingMetadata.out | 6 + .../testFilterPushDown.out | 6 + .../testLimitPushDown.out | 9 + .../testPartitionPushDown.out | 6 + .../testProjectPushDown.out | 6 + .../testReadingMetadata.out | 6 + .../testWatermarkPushDown.out | 6 + .../testJoinTemporalFunction.out | 6 + .../testTemporalTableJoin.out | 6 + .../testSortProcessingTime.out | 6 + .../testSortRowTime.out | 6 + .../UnionJsonPlanTest_jsonplan/testUnion.out | 6 + .../ValuesJsonPlanTest_jsonplan/testValues.out | 6 + .../testWatermarkAssigner.out | 6 + .../testDistinctSplitEnabled.out | 18 ++ .../testEventTimeCumulateWindow.out | 12 + .../testEventTimeCumulateWindowWithOffset.out | 12 + .../testEventTimeHopWindow.out | 12 + .../testEventTimeHopWindowWithOffset.out | 12 + .../testEventTimeTumbleWindow.out | 12 + .../testEventTimeTumbleWindowWithOffset.out | 12 + .../testProcTimeCumulateWindow.out | 9 + .../testProcTimeHopWindow.out | 9 + .../testProcTimeTumbleWindow.out | 9 + .../testEventTimeTumbleWindow.out | 21 ++ .../testFollowedByWindowDeduplicate.out | 12 + .../testFollowedByWindowJoin.out | 15 ++ .../testFollowedByWindowRank.out | 12 + .../testIndividualWindowTVF.out | 9 + .../testIndividualWindowTVFProcessingTime.out | 9 + 271 files changed, 1965 insertions(+), 393 deletions(-) copy flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/serde/{ExecNodeGraphJsonDeserializer.java => ConfigurationJsonDeserializer.java} (65%) copy flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/serde/{ObjectIdentifierJsonSerializer.java => ConfigurationJsonSerializer.java} (68%) copy flink-yarn/src/main/java/org/apache/flink/yarn/cli/FallbackYarnSessionCli.java => flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/serde/ConfigurationJsonSerializerFilter.java (59%)