This is an automated email from the ASF dual-hosted git repository.
chengpan pushed a change to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-celeborn.git
from f027a25db [CELEBORN-924] Change log level to error in
PartitionFilesSorter upon Exception
add 2c07c55e7 [CELEBORN-919] Move Columnar Shuffle code into an individual
module
No new revisions were added by this update.
Summary of changes:
.../{common => spark-3-columnar-shuffle}/pom.xml | 28 ++--
.../celeborn/ColumnarHashBasedShuffleWriter.java | 144 +++++++++++++++++++
.../celeborn/CustomShuffleDependencyUtils.java} | 34 +++--
.../columnar/CelebornColumnDictionary.java | 0
.../celeborn/CelebornColumnarShuffleReader.scala | 63 +++++++++
.../execution/columnar/CelebornBatchBuilder.scala | 0
.../columnar/CelebornColumnAccessor.scala | 0
.../execution/columnar/CelebornColumnBuilder.scala | 0
.../execution/columnar/CelebornColumnStats.scala | 0
.../execution/columnar/CelebornColumnType.scala | 0
.../columnar/CelebornColumnarBatchBuilder.scala | 0
.../CelebornColumnarBatchCodeGenBuild.scala | 0
.../columnar/CelebornColumnarBatchSerializer.scala | 0
.../CelebornCompressibleColumnAccessor.scala | 0
.../CelebornCompressibleColumnBuilder.scala | 0
.../columnar/CelebornCompressionScheme.scala | 0
.../columnar/CelebornCompressionSchemes.scala | 0
.../columnar/CelebornNullableColumnAccessor.scala | 0
.../columnar/CelebornNullableColumnBuilder.scala | 0
.../ColumnarHashBasedShuffleWriterSuiteJ.java | 75 ++++++++++
.../src/test/resources/log4j.properties | 0
.../src/test/resources/log4j2-test.xml | 0
.../CelebornColumnarShuffleReaderSuite.scala | 59 ++++++++
client-spark/spark-3-shaded/pom.xml | 53 +++++++
.../shuffle/celeborn/HashBasedShuffleWriter.java | 157 ++++-----------------
.../shuffle/celeborn/SparkShuffleManager.java | 94 +++++++-----
.../apache/spark/shuffle/celeborn/SparkUtils.java | 73 ++++++++--
.../shuffle/celeborn/CelebornShuffleReader.scala | 26 ++--
.../celeborn/CelebornShuffleWriterSuiteBase.java | 7 +-
pom.xml | 11 ++
project/CelebornBuild.scala | 55 +++++++-
31 files changed, 651 insertions(+), 228 deletions(-)
copy client-spark/{common => spark-3-columnar-shuffle}/pom.xml (80%)
create mode 100644
client-spark/spark-3-columnar-shuffle/src/main/java/org/apache/spark/shuffle/celeborn/ColumnarHashBasedShuffleWriter.java
copy
client-spark/{spark-2/src/test/java/org/apache/spark/shuffle/celeborn/SortBasedShuffleWriterSuiteJ.java
=>
spark-3-columnar-shuffle/src/main/java/org/apache/spark/shuffle/celeborn/CustomShuffleDependencyUtils.java}
(53%)
rename client-spark/{spark-3 =>
spark-3-columnar-shuffle}/src/main/java/org/apache/spark/sql/execution/columnar/CelebornColumnDictionary.java
(100%)
create mode 100644
client-spark/spark-3-columnar-shuffle/src/main/scala/org/apache/spark/shuffle/celeborn/CelebornColumnarShuffleReader.scala
rename client-spark/{spark-3 =>
spark-3-columnar-shuffle}/src/main/scala/org/apache/spark/sql/execution/columnar/CelebornBatchBuilder.scala
(100%)
rename client-spark/{spark-3 =>
spark-3-columnar-shuffle}/src/main/scala/org/apache/spark/sql/execution/columnar/CelebornColumnAccessor.scala
(100%)
rename client-spark/{spark-3 =>
spark-3-columnar-shuffle}/src/main/scala/org/apache/spark/sql/execution/columnar/CelebornColumnBuilder.scala
(100%)
rename client-spark/{spark-3 =>
spark-3-columnar-shuffle}/src/main/scala/org/apache/spark/sql/execution/columnar/CelebornColumnStats.scala
(100%)
rename client-spark/{spark-3 =>
spark-3-columnar-shuffle}/src/main/scala/org/apache/spark/sql/execution/columnar/CelebornColumnType.scala
(100%)
rename client-spark/{spark-3 =>
spark-3-columnar-shuffle}/src/main/scala/org/apache/spark/sql/execution/columnar/CelebornColumnarBatchBuilder.scala
(100%)
rename client-spark/{spark-3 =>
spark-3-columnar-shuffle}/src/main/scala/org/apache/spark/sql/execution/columnar/CelebornColumnarBatchCodeGenBuild.scala
(100%)
rename client-spark/{spark-3 =>
spark-3-columnar-shuffle}/src/main/scala/org/apache/spark/sql/execution/columnar/CelebornColumnarBatchSerializer.scala
(100%)
rename client-spark/{spark-3 =>
spark-3-columnar-shuffle}/src/main/scala/org/apache/spark/sql/execution/columnar/CelebornCompressibleColumnAccessor.scala
(100%)
rename client-spark/{spark-3 =>
spark-3-columnar-shuffle}/src/main/scala/org/apache/spark/sql/execution/columnar/CelebornCompressibleColumnBuilder.scala
(100%)
rename client-spark/{spark-3 =>
spark-3-columnar-shuffle}/src/main/scala/org/apache/spark/sql/execution/columnar/CelebornCompressionScheme.scala
(100%)
rename client-spark/{spark-3 =>
spark-3-columnar-shuffle}/src/main/scala/org/apache/spark/sql/execution/columnar/CelebornCompressionSchemes.scala
(100%)
rename client-spark/{spark-3 =>
spark-3-columnar-shuffle}/src/main/scala/org/apache/spark/sql/execution/columnar/CelebornNullableColumnAccessor.scala
(100%)
rename client-spark/{spark-3 =>
spark-3-columnar-shuffle}/src/main/scala/org/apache/spark/sql/execution/columnar/CelebornNullableColumnBuilder.scala
(100%)
create mode 100644
client-spark/spark-3-columnar-shuffle/src/test/java/org/apache/spark/shuffle/celeborn/ColumnarHashBasedShuffleWriterSuiteJ.java
copy client-spark/{common =>
spark-3-columnar-shuffle}/src/test/resources/log4j.properties (100%)
copy {client-flink/flink-1.14 =>
client-spark/spark-3-columnar-shuffle}/src/test/resources/log4j2-test.xml (100%)
create mode 100644
client-spark/spark-3-columnar-shuffle/src/test/scala/org/apache/spark/shuffle/celeborn/CelebornColumnarShuffleReaderSuite.scala
