This is an automated email from the ASF dual-hosted git repository. liyuanjian pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/spark.git.
from 21db916 [SPARK-38484][PYTHON] Move usage logging instrumentation util functions from pandas module to pyspark.util module add 4e31000 [SPARK-38204][SS] Use StatefulOpClusteredDistribution for stateful operators with respecting backward compatibility No new revisions were added by this update. Summary of changes: docs/ss-migration-guide.md | 4 + .../org/apache/spark/sql/internal/SQLConf.scala | 17 + .../spark/sql/execution/aggregate/AggUtils.scala | 64 ++- .../execution/aggregate/BaseAggregateExec.scala | 23 +- .../execution/aggregate/HashAggregateExec.scala | 2 + .../execution/aggregate/MergingSessionsExec.scala | 15 +- .../aggregate/ObjectHashAggregateExec.scala | 2 + .../execution/aggregate/SortAggregateExec.scala | 2 + .../execution/aggregate/UpdatingSessionsExec.scala | 18 +- .../streaming/FlatMapGroupsWithStateExec.scala | 13 +- .../execution/streaming/IncrementalExecution.scala | 17 + .../spark/sql/execution/streaming/OffsetSeq.scala | 5 +- .../streaming/StatefulOperatorPartitioning.scala | 53 +++ .../execution/streaming/statefulOperators.scala | 37 +- .../commits/0 | 0 .../metadata | 1 + .../offsets/0 | 3 + .../state/0/0}/1.delta | Bin .../state/0/0/_metadata/schema | Bin 0 -> 407 bytes .../state/0/1/1.delta | Bin 0 -> 96 bytes .../state/0/2}/1.delta | Bin .../state/0/3}/1.delta | Bin .../state/0/4/1.delta | Bin .../commits/.0.crc | Bin 0 -> 12 bytes .../commits/.1.crc | Bin 0 -> 12 bytes .../commits/0 | 0 .../commits/1 | 0 .../metadata | 1 + .../offsets/.0.crc | Bin 0 -> 16 bytes .../offsets/.1.crc | Bin 0 -> 16 bytes .../offsets/0 | 3 + .../offsets/1 | 3 + .../state/0/0/.1.delta.crc | Bin 0 -> 12 bytes .../state/0/0/.2.delta.crc | Bin 0 -> 12 bytes .../state/0/0}/1.delta | Bin .../state/0/0}/2.delta | Bin .../state/0/0/_metadata/.schema.crc | Bin 0 -> 12 bytes .../state/0/0/_metadata/schema | Bin 0 -> 393 bytes .../state/0/1/.1.delta.crc | Bin 0 -> 12 bytes .../state/0/1/.2.delta.crc | Bin 0 -> 12 bytes .../state/0/1}/1.delta | Bin .../state/0/1/2.delta | Bin 0 -> 75 bytes .../state/0/2/.1.delta.crc | Bin 0 -> 12 bytes .../state/0/2/.2.delta.crc | Bin 0 -> 12 bytes .../state/0/2}/1.delta | Bin .../state/0/2/2.delta | Bin 0 -> 75 bytes .../state/0/3/.1.delta.crc | Bin 0 -> 12 bytes .../state/0/3/.2.delta.crc | Bin 0 -> 12 bytes .../state/0/3/1.delta | Bin 0 -> 74 bytes .../state/0/3/2.delta | Bin .../state/0/4/.1.delta.crc | Bin 0 -> 12 bytes .../state/0/4/.2.delta.crc | Bin 0 -> 12 bytes .../state/0/4/1.delta | Bin 0 -> 75 bytes .../state/0/4}/2.delta | Bin .../commits/.0.crc | Bin 0 -> 12 bytes .../commits/0 | 0 .../metadata | 1 + .../offsets/.0.crc | Bin 0 -> 16 bytes .../offsets/0 | 3 + .../state/0/0/.1.delta.crc | Bin 0 -> 12 bytes .../state/0/0}/1.delta | Bin .../state/0/0/_metadata/.schema.crc | Bin 0 -> 12 bytes .../state/0/0/_metadata/schema | Bin 0 -> 415 bytes .../state/0/1/.1.delta.crc | Bin 0 -> 12 bytes .../state/0/1/1.delta | Bin 0 -> 138 bytes .../state/0/2/.1.delta.crc | Bin 0 -> 12 bytes .../state/0/2}/1.delta | Bin .../state/0/3/.1.delta.crc | Bin 0 -> 12 bytes .../state/0/3}/1.delta | Bin .../state/0/4/.1.delta.crc | Bin 0 -> 12 bytes .../state/0/4/1.delta | Bin .../commits/.0.crc | Bin 0 -> 12 bytes .../commits/0 | 0 .../metadata | 1 + .../offsets/.0.crc | Bin 0 -> 16 bytes .../offsets/0 | 3 + .../state/0/0/.1.delta.crc | Bin 0 -> 12 bytes .../state/0/0}/1.delta | Bin .../state/0/0/_metadata/.schema.crc | Bin 0 -> 12 bytes .../state/0/0/_metadata/schema | Bin 0 -> 415 bytes .../state/0/1/.1.delta.crc | Bin 0 -> 12 bytes .../state/0/1/1.delta | Bin 0 -> 96 bytes .../state/0/2/.1.delta.crc | Bin 0 -> 12 bytes .../state/0/2}/1.delta | Bin .../state/0/3/.1.delta.crc | Bin 0 -> 12 bytes .../state/0/3}/1.delta | Bin .../state/0/4/.1.delta.crc | Bin 0 -> 12 bytes .../state/0/4/1.delta | Bin .../commits/.0.crc | Bin 0 -> 12 bytes .../commits/.1.crc | Bin 0 -> 12 bytes .../commits/0 | 2 + .../commits/1 | 2 + .../metadata | 1 + .../offsets/.0.crc | Bin 0 -> 16 bytes .../offsets/.1.crc | Bin 0 -> 16 bytes .../offsets/0 | 3 + .../offsets/1 | 3 + .../state/0/0/.1.delta.crc | Bin 0 -> 12 bytes .../state/0/0/.2.delta.crc | Bin 0 -> 12 bytes .../state/0/0}/1.delta | Bin .../state/0/0}/2.delta | Bin .../state/0/0/_metadata/.schema.crc | Bin 0 -> 16 bytes .../state/0/0/_metadata/schema | Bin 0 -> 754 bytes .../state/0/1/.1.delta.crc | Bin 0 -> 12 bytes .../state/0/1/.2.delta.crc | Bin 0 -> 12 bytes .../state/0/1/1.delta | Bin 0 -> 259 bytes .../state/0/1}/2.delta | Bin .../state/0/2/.1.delta.crc | Bin 0 -> 12 bytes .../state/0/2/.2.delta.crc | Bin 0 -> 12 bytes .../state/0/2}/1.delta | Bin .../state/0/2}/2.delta | Bin .../state/0/3/.1.delta.crc | Bin 0 -> 12 bytes .../state/0/3/.2.delta.crc | Bin 0 -> 12 bytes .../state/0/3/1.delta | Bin 0 -> 230 bytes .../state/0/3/2.delta | Bin .../state/0/4/.1.delta.crc | Bin 0 -> 12 bytes .../state/0/4/.2.delta.crc | Bin 0 -> 12 bytes .../state/0/4/1.delta | Bin .../state/0/4}/2.delta | Bin .../commits/.0.crc | Bin 0 -> 12 bytes .../commits/.1.crc | Bin 0 -> 12 bytes .../commits/0 | 0 .../commits/1 | 0 .../metadata | 1 + .../offsets/.0.crc | Bin 0 -> 16 bytes .../offsets/.1.crc | Bin 0 -> 16 bytes .../offsets/0 | 3 + .../offsets/1 | 3 + .../state/0/0/.1.delta.crc | Bin 0 -> 12 bytes .../state/0/0/.2.delta.crc | Bin 0 -> 12 bytes .../state/0/0}/1.delta | Bin .../state/0/0}/2.delta | Bin .../state/0/0/_metadata/.schema.crc | Bin 0 -> 12 bytes .../state/0/0/_metadata/schema | Bin 0 -> 262 bytes .../state/0/1/.1.delta.crc | Bin 0 -> 12 bytes .../state/0/1/.2.delta.crc | Bin 0 -> 12 bytes .../state/0/1/1.delta | Bin 0 -> 82 bytes .../state/0/1/2.delta | Bin 0 -> 82 bytes .../state/0/2/.1.delta.crc | Bin 0 -> 12 bytes .../state/0/2/.2.delta.crc | Bin 0 -> 12 bytes .../state/0/2}/1.delta | Bin .../state/0/2}/2.delta | Bin .../state/0/3/.1.delta.crc | Bin 0 -> 12 bytes .../state/0/3/.2.delta.crc | Bin 0 -> 12 bytes .../state/0/3}/1.delta | Bin .../state/0/3/2.delta | Bin .../state/0/4/.1.delta.crc | Bin 0 -> 12 bytes .../state/0/4/.2.delta.crc | Bin 0 -> 12 bytes .../state/0/4/1.delta | Bin .../state/0/4/2.delta | Bin 0 -> 82 bytes .../sql/execution/WholeStageCodegenSuite.scala | 2 +- .../FlatMapGroupsWithStateDistributionSuite.scala | 455 +++++++++++++++++++++ .../StreamingAggregationDistributionSuite.scala | 223 ++++++++++ .../sql/streaming/StreamingAggregationSuite.scala | 37 +- .../StreamingDeduplicationDistributionSuite.scala | 148 +++++++ .../StreamingSessionWindowDistributionSuite.scala | 225 ++++++++++ ...StatefulOpClusteredDistributionTestHelper.scala | 80 ++++ 157 files changed, 1393 insertions(+), 86 deletions(-) create mode 100644 sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/StatefulOperatorPartitioning.scala copy sql/core/src/test/resources/structured-streaming/{escaped-path-2.4.0/chk%252520%252525@%252523chk => checkpoint-version-3.1.0-flatmapgroupswithstate-repartition}/commits/0 (100%) create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.1.0-flatmapgroupswithstate-repartition/metadata create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.1.0-flatmapgroupswithstate-repartition/offsets/0 copy sql/core/src/test/resources/structured-streaming/{checkpoint-version-2.4.5-for-compatibility-test-sum-decimal/state/0/4 => checkpoint-version-3.1.0-flatmapgroupswithstate-repartition/state/0/0}/1.delta (100%) create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.1.0-flatmapgroupswithstate-repartition/state/0/0/_metadata/schema create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.1.0-flatmapgroupswithstate-repartition/state/0/1/1.delta copy sql/core/src/test/resources/structured-streaming/{checkpoint-version-2.4.5-for-compatibility-test-sum-decimal/state/0/4 => checkpoint-version-3.1.0-flatmapgroupswithstate-repartition/state/0/2}/1.delta (100%) copy sql/core/src/test/resources/structured-streaming/{checkpoint-version-2.4.5-for-compatibility-test-sum-decimal/state/0/4 => checkpoint-version-3.1.0-flatmapgroupswithstate-repartition/state/0/3}/1.delta (100%) copy sql/core/src/test/resources/structured-streaming/{checkpoint-version-2.4.5-for-compatibility-test-sum-decimal => checkpoint-version-3.1.0-flatmapgroupswithstate-repartition}/state/0/4/1.delta (100%) create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-deduplication-with-repartition/commits/.0.crc create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-deduplication-with-repartition/commits/.1.crc copy sql/core/src/test/resources/structured-streaming/{escaped-path-2.4.0/chk%252520%252525@%252523chk => checkpoint-version-3.2.0-deduplication-with-repartition}/commits/0 (100%) copy sql/core/src/test/resources/structured-streaming/{checkpoint-test-offsetId-commitId-inconsistent => checkpoint-version-3.2.0-deduplication-with-repartition}/commits/1 (100%) create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-deduplication-with-repartition/metadata create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-deduplication-with-repartition/offsets/.0.crc create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-deduplication-with-repartition/offsets/.1.crc create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-deduplication-with-repartition/offsets/0 create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-deduplication-with-repartition/offsets/1 create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-deduplication-with-repartition/state/0/0/.1.delta.crc create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-deduplication-with-repartition/state/0/0/.2.delta.crc copy sql/core/src/test/resources/structured-streaming/{checkpoint-version-2.4.5-for-compatibility-test-sum-decimal/state/0/4 => checkpoint-version-3.2.0-deduplication-with-repartition/state/0/0}/1.delta (100%) copy sql/core/src/test/resources/structured-streaming/{checkpoint-version-2.3.1-without-commit-log-metadata/state/0/3 => checkpoint-version-3.2.0-deduplication-with-repartition/state/0/0}/2.delta (100%) create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-deduplication-with-repartition/state/0/0/_metadata/.schema.crc create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-deduplication-with-repartition/state/0/0/_metadata/schema create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-deduplication-with-repartition/state/0/1/.1.delta.crc create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-deduplication-with-repartition/state/0/1/.2.delta.crc copy sql/core/src/test/resources/structured-streaming/{checkpoint-version-2.4.5-for-compatibility-test-sum-decimal/state/0/4 => checkpoint-version-3.2.0-deduplication-with-repartition/state/0/1}/1.delta (100%) create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-deduplication-with-repartition/state/0/1/2.delta create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-deduplication-with-repartition/state/0/2/.1.delta.crc create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-deduplication-with-repartition/state/0/2/.2.delta.crc copy sql/core/src/test/resources/structured-streaming/{checkpoint-version-2.4.5-for-compatibility-test-sum-decimal/state/0/4 => checkpoint-version-3.2.0-deduplication-with-repartition/state/0/2}/1.delta (100%) create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-deduplication-with-repartition/state/0/2/2.delta create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-deduplication-with-repartition/state/0/3/.1.delta.crc create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-deduplication-with-repartition/state/0/3/.2.delta.crc create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-deduplication-with-repartition/state/0/3/1.delta copy sql/core/src/test/resources/structured-streaming/{checkpoint-version-2.3.1-without-commit-log-metadata => checkpoint-version-3.2.0-deduplication-with-repartition}/state/0/3/2.delta (100%) create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-deduplication-with-repartition/state/0/4/.1.delta.crc create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-deduplication-with-repartition/state/0/4/.2.delta.crc create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-deduplication-with-repartition/state/0/4/1.delta copy sql/core/src/test/resources/structured-streaming/{checkpoint-version-2.3.1-without-commit-log-metadata/state/0/3 => checkpoint-version-3.2.0-deduplication-with-repartition/state/0/4}/2.delta (100%) create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-flatmapgroupswithstate1-repartition/commits/.0.crc copy sql/core/src/test/resources/structured-streaming/{escaped-path-2.4.0/chk%252520%252525@%252523chk => checkpoint-version-3.2.0-flatmapgroupswithstate1-repartition}/commits/0 (100%) create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-flatmapgroupswithstate1-repartition/metadata create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-flatmapgroupswithstate1-repartition/offsets/.0.crc create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-flatmapgroupswithstate1-repartition/offsets/0 create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-flatmapgroupswithstate1-repartition/state/0/0/.1.delta.crc copy sql/core/src/test/resources/structured-streaming/{checkpoint-version-2.4.5-for-compatibility-test-sum-decimal/state/0/4 => checkpoint-version-3.2.0-flatmapgroupswithstate1-repartition/state/0/0}/1.delta (100%) create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-flatmapgroupswithstate1-repartition/state/0/0/_metadata/.schema.crc create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-flatmapgroupswithstate1-repartition/state/0/0/_metadata/schema create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-flatmapgroupswithstate1-repartition/state/0/1/.1.delta.crc create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-flatmapgroupswithstate1-repartition/state/0/1/1.delta create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-flatmapgroupswithstate1-repartition/state/0/2/.1.delta.crc copy sql/core/src/test/resources/structured-streaming/{checkpoint-version-2.4.5-for-compatibility-test-sum-decimal/state/0/4 => checkpoint-version-3.2.0-flatmapgroupswithstate1-repartition/state/0/2}/1.delta (100%) create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-flatmapgroupswithstate1-repartition/state/0/3/.1.delta.crc copy sql/core/src/test/resources/structured-streaming/{checkpoint-version-2.4.5-for-compatibility-test-sum-decimal/state/0/4 => checkpoint-version-3.2.0-flatmapgroupswithstate1-repartition/state/0/3}/1.delta (100%) create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-flatmapgroupswithstate1-repartition/state/0/4/.1.delta.crc copy sql/core/src/test/resources/structured-streaming/{checkpoint-version-2.4.5-for-compatibility-test-sum-decimal => checkpoint-version-3.2.0-flatmapgroupswithstate1-repartition}/state/0/4/1.delta (100%) create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-flatmapgroupswithstate2-repartition/commits/.0.crc copy sql/core/src/test/resources/structured-streaming/{escaped-path-2.4.0/chk%252520%252525@%252523chk => checkpoint-version-3.2.0-flatmapgroupswithstate2-repartition}/commits/0 (100%) create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-flatmapgroupswithstate2-repartition/metadata create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-flatmapgroupswithstate2-repartition/offsets/.0.crc create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-flatmapgroupswithstate2-repartition/offsets/0 create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-flatmapgroupswithstate2-repartition/state/0/0/.1.delta.crc copy sql/core/src/test/resources/structured-streaming/{checkpoint-version-2.4.5-for-compatibility-test-sum-decimal/state/0/4 => checkpoint-version-3.2.0-flatmapgroupswithstate2-repartition/state/0/0}/1.delta (100%) create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-flatmapgroupswithstate2-repartition/state/0/0/_metadata/.schema.crc create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-flatmapgroupswithstate2-repartition/state/0/0/_metadata/schema create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-flatmapgroupswithstate2-repartition/state/0/1/.1.delta.crc create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-flatmapgroupswithstate2-repartition/state/0/1/1.delta create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-flatmapgroupswithstate2-repartition/state/0/2/.1.delta.crc copy sql/core/src/test/resources/structured-streaming/{checkpoint-version-2.4.5-for-compatibility-test-sum-decimal/state/0/4 => checkpoint-version-3.2.0-flatmapgroupswithstate2-repartition/state/0/2}/1.delta (100%) create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-flatmapgroupswithstate2-repartition/state/0/3/.1.delta.crc copy sql/core/src/test/resources/structured-streaming/{checkpoint-version-2.4.5-for-compatibility-test-sum-decimal/state/0/4 => checkpoint-version-3.2.0-flatmapgroupswithstate2-repartition/state/0/3}/1.delta (100%) create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-flatmapgroupswithstate2-repartition/state/0/4/.1.delta.crc copy sql/core/src/test/resources/structured-streaming/{checkpoint-version-2.4.5-for-compatibility-test-sum-decimal => checkpoint-version-3.2.0-flatmapgroupswithstate2-repartition}/state/0/4/1.delta (100%) create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-session-window-with-repartition/commits/.0.crc create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-session-window-with-repartition/commits/.1.crc create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-session-window-with-repartition/commits/0 create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-session-window-with-repartition/commits/1 create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-session-window-with-repartition/metadata create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-session-window-with-repartition/offsets/.0.crc create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-session-window-with-repartition/offsets/.1.crc create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-session-window-with-repartition/offsets/0 create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-session-window-with-repartition/offsets/1 create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-session-window-with-repartition/state/0/0/.1.delta.crc create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-session-window-with-repartition/state/0/0/.2.delta.crc copy sql/core/src/test/resources/structured-streaming/{checkpoint-version-2.4.5-for-compatibility-test-sum-decimal/state/0/4 => checkpoint-version-3.2.0-session-window-with-repartition/state/0/0}/1.delta (100%) copy sql/core/src/test/resources/structured-streaming/{checkpoint-version-2.3.1-without-commit-log-metadata/state/0/3 => checkpoint-version-3.2.0-session-window-with-repartition/state/0/0}/2.delta (100%) create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-session-window-with-repartition/state/0/0/_metadata/.schema.crc create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-session-window-with-repartition/state/0/0/_metadata/schema create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-session-window-with-repartition/state/0/1/.1.delta.crc create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-session-window-with-repartition/state/0/1/.2.delta.crc create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-session-window-with-repartition/state/0/1/1.delta copy sql/core/src/test/resources/structured-streaming/{checkpoint-version-2.3.1-without-commit-log-metadata/state/0/3 => checkpoint-version-3.2.0-session-window-with-repartition/state/0/1}/2.delta (100%) create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-session-window-with-repartition/state/0/2/.1.delta.crc create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-session-window-with-repartition/state/0/2/.2.delta.crc copy sql/core/src/test/resources/structured-streaming/{checkpoint-version-2.4.5-for-compatibility-test-sum-decimal/state/0/4 => checkpoint-version-3.2.0-session-window-with-repartition/state/0/2}/1.delta (100%) copy sql/core/src/test/resources/structured-streaming/{checkpoint-version-2.3.1-without-commit-log-metadata/state/0/3 => checkpoint-version-3.2.0-session-window-with-repartition/state/0/2}/2.delta (100%) create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-session-window-with-repartition/state/0/3/.1.delta.crc create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-session-window-with-repartition/state/0/3/.2.delta.crc create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-session-window-with-repartition/state/0/3/1.delta copy sql/core/src/test/resources/structured-streaming/{checkpoint-version-2.3.1-without-commit-log-metadata => checkpoint-version-3.2.0-session-window-with-repartition}/state/0/3/2.delta (100%) create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-session-window-with-repartition/state/0/4/.1.delta.crc create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-session-window-with-repartition/state/0/4/.2.delta.crc copy sql/core/src/test/resources/structured-streaming/{checkpoint-version-2.4.5-for-compatibility-test-sum-decimal => checkpoint-version-3.2.0-session-window-with-repartition}/state/0/4/1.delta (100%) copy sql/core/src/test/resources/structured-streaming/{checkpoint-version-2.3.1-without-commit-log-metadata/state/0/3 => checkpoint-version-3.2.0-session-window-with-repartition/state/0/4}/2.delta (100%) create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-streaming-aggregate-with-repartition/commits/.0.crc create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-streaming-aggregate-with-repartition/commits/.1.crc copy sql/core/src/test/resources/structured-streaming/{escaped-path-2.4.0/chk%252520%252525@%252523chk => checkpoint-version-3.2.0-streaming-aggregate-with-repartition}/commits/0 (100%) copy sql/core/src/test/resources/structured-streaming/{checkpoint-test-offsetId-commitId-inconsistent => checkpoint-version-3.2.0-streaming-aggregate-with-repartition}/commits/1 (100%) create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-streaming-aggregate-with-repartition/metadata create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-streaming-aggregate-with-repartition/offsets/.0.crc create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-streaming-aggregate-with-repartition/offsets/.1.crc create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-streaming-aggregate-with-repartition/offsets/0 create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-streaming-aggregate-with-repartition/offsets/1 create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-streaming-aggregate-with-repartition/state/0/0/.1.delta.crc create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-streaming-aggregate-with-repartition/state/0/0/.2.delta.crc copy sql/core/src/test/resources/structured-streaming/{checkpoint-version-2.4.5-for-compatibility-test-sum-decimal/state/0/4 => checkpoint-version-3.2.0-streaming-aggregate-with-repartition/state/0/0}/1.delta (100%) copy sql/core/src/test/resources/structured-streaming/{checkpoint-version-2.3.1-without-commit-log-metadata/state/0/3 => checkpoint-version-3.2.0-streaming-aggregate-with-repartition/state/0/0}/2.delta (100%) create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-streaming-aggregate-with-repartition/state/0/0/_metadata/.schema.crc create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-streaming-aggregate-with-repartition/state/0/0/_metadata/schema create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-streaming-aggregate-with-repartition/state/0/1/.1.delta.crc create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-streaming-aggregate-with-repartition/state/0/1/.2.delta.crc create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-streaming-aggregate-with-repartition/state/0/1/1.delta create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-streaming-aggregate-with-repartition/state/0/1/2.delta create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-streaming-aggregate-with-repartition/state/0/2/.1.delta.crc create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-streaming-aggregate-with-repartition/state/0/2/.2.delta.crc copy sql/core/src/test/resources/structured-streaming/{checkpoint-version-2.4.5-for-compatibility-test-sum-decimal/state/0/4 => checkpoint-version-3.2.0-streaming-aggregate-with-repartition/state/0/2}/1.delta (100%) copy sql/core/src/test/resources/structured-streaming/{checkpoint-version-2.3.1-without-commit-log-metadata/state/0/3 => checkpoint-version-3.2.0-streaming-aggregate-with-repartition/state/0/2}/2.delta (100%) create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-streaming-aggregate-with-repartition/state/0/3/.1.delta.crc create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-streaming-aggregate-with-repartition/state/0/3/.2.delta.crc copy sql/core/src/test/resources/structured-streaming/{checkpoint-version-2.4.5-for-compatibility-test-sum-decimal/state/0/4 => checkpoint-version-3.2.0-streaming-aggregate-with-repartition/state/0/3}/1.delta (100%) copy sql/core/src/test/resources/structured-streaming/{checkpoint-version-2.3.1-without-commit-log-metadata => checkpoint-version-3.2.0-streaming-aggregate-with-repartition}/state/0/3/2.delta (100%) create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-streaming-aggregate-with-repartition/state/0/4/.1.delta.crc create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-streaming-aggregate-with-repartition/state/0/4/.2.delta.crc copy sql/core/src/test/resources/structured-streaming/{checkpoint-version-2.4.5-for-compatibility-test-sum-decimal => checkpoint-version-3.2.0-streaming-aggregate-with-repartition}/state/0/4/1.delta (100%) create mode 100644 sql/core/src/test/resources/structured-streaming/checkpoint-version-3.2.0-streaming-aggregate-with-repartition/state/0/4/2.delta create mode 100644 sql/core/src/test/scala/org/apache/spark/sql/streaming/FlatMapGroupsWithStateDistributionSuite.scala create mode 100644 sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamingAggregationDistributionSuite.scala create mode 100644 sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamingDeduplicationDistributionSuite.scala create mode 100644 sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamingSessionWindowDistributionSuite.scala create mode 100644 sql/core/src/test/scala/org/apache/spark/sql/streaming/util/StatefulOpClusteredDistributionTestHelper.scala --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org