This is an automated email from the ASF dual-hosted git repository.
rexxiong pushed a change to branch main
in repository https://gitbox.apache.org/repos/asf/celeborn.git
from 4b60dae0f [CELEBORN-1789][DOC] Document on Java Columnar Shuffle
add fde6365f6 [CELEBORN-1413] Support Spark 4.0
No new revisions were added by this update.
Summary of changes:
.github/workflows/maven.yml | 6 +++-
.../spark/shuffle/celeborn/SparkCommonUtils.java | 6 ++--
client-spark/{spark-3 => spark-3-4}/pom.xml | 4 +--
.../shuffle/celeborn/CelebornShuffleDataIO.java | 0
.../shuffle/celeborn/HashBasedShuffleWriter.java | 0
.../shuffle/celeborn/SortBasedShuffleWriter.java | 0
.../shuffle/celeborn/SparkShuffleManager.java | 0
.../apache/spark/shuffle/celeborn/SparkUtils.java | 0
.../scala/org/apache/spark/SparkVersionUtil.scala | 0
.../spark/celeborn/ExceptionMakerHelper.scala | 0
.../CelebornShuffleFallbackPolicyRunner.scala | 0
.../shuffle/celeborn/CelebornShuffleHandle.scala | 0
.../shuffle/celeborn/CelebornShuffleReader.scala | 0
.../celeborn/CelebornShuffleWriterSuiteBase.java | 0
.../celeborn/HashBasedShuffleWriterSuiteJ.java | 0
.../spark/shuffle/celeborn/ShuffleManagerHook.java | 0
.../celeborn/SortBasedShuffleWriterSuiteJ.java | 0
.../celeborn/TestCelebornShuffleManager.java | 0
.../src/test/resources/log4j.properties | 0
.../src/test/resources/log4j2-test.xml | 0
.../celeborn/CelebornShuffleManagerSuite.scala | 0
client-spark/spark-3-columnar-common/pom.xml | 2 +-
client-spark/spark-3-columnar-shuffle/pom.xml | 2 +-
client-spark/spark-3-shaded/pom.xml | 2 +-
.../pom.xml | 4 +--
.../celeborn/ColumnarHashBasedShuffleWriter.java | 0
.../celeborn/CelebornColumnarShuffleReader.scala | 0
.../columnar/CelebornColumnAccessor.scala | 0
.../execution/columnar/CelebornColumnBuilder.scala | 2 +-
.../execution/columnar/CelebornColumnStats.scala | 0
.../execution/columnar/CelebornColumnType.scala | 6 +++-
.../columnar/CelebornColumnarBatchBuilder.scala | 0
.../CelebornColumnarBatchCodeGenBuild.scala | 0
.../columnar/CelebornColumnarBatchSerializer.scala | 0
.../CelebornCompressibleColumnAccessor.scala | 0
.../CelebornCompressibleColumnBuilder.scala | 0
.../columnar/CelebornCompressionScheme.scala | 0
.../columnar/CelebornCompressionSchemes.scala | 0
.../columnar/CelebornNullableColumnAccessor.scala | 0
.../columnar/CelebornNullableColumnBuilder.scala | 0
.../ColumnarHashBasedShuffleWriterSuiteJ.java | 0
.../src/test/resources/log4j2-test.xml | 0
.../CelebornColumnarShuffleReaderSuite.scala | 0
.../spark-4-shaded}/pom.xml | 12 +++++--
.../src/main/resources/META-INF/LICENSE | 0
.../src/main/resources/META-INF/NOTICE | 0
.../META-INF/licenses/LICENSE-protobuf.txt | 0
.../common/metrics/source/AbstractSource.scala | 2 +-
.../metrics/source/CelebornSourceSuite.scala | 4 +--
dev/dependencies.sh | 2 +-
dev/reformat | 2 ++
.../deploy/master/http/api/v1/WorkerResource.scala | 4 +--
pom.xml | 32 +++++++++++++----
project/CelebornBuild.scala | 33 +++++++++++++----
tests/spark-it/pom.xml | 42 +++++++++++++++-------
.../service/deploy/worker/PushDataHandler.scala | 4 +--
56 files changed, 120 insertions(+), 51 deletions(-)
rename client-spark/{spark-3 => spark-3-4}/pom.xml (96%)
rename client-spark/{spark-3 =>
spark-3-4}/src/main/java/org/apache/spark/shuffle/celeborn/CelebornShuffleDataIO.java
(100%)
rename client-spark/{spark-3 =>
spark-3-4}/src/main/java/org/apache/spark/shuffle/celeborn/HashBasedShuffleWriter.java
(100%)
rename client-spark/{spark-3 =>
spark-3-4}/src/main/java/org/apache/spark/shuffle/celeborn/SortBasedShuffleWriter.java
(100%)
rename client-spark/{spark-3 =>
spark-3-4}/src/main/java/org/apache/spark/shuffle/celeborn/SparkShuffleManager.java
(100%)
rename client-spark/{spark-3 =>
spark-3-4}/src/main/java/org/apache/spark/shuffle/celeborn/SparkUtils.java
(100%)
rename client-spark/{spark-3 =>
spark-3-4}/src/main/scala/org/apache/spark/SparkVersionUtil.scala (100%)
rename client-spark/{spark-3 =>
spark-3-4}/src/main/scala/org/apache/spark/celeborn/ExceptionMakerHelper.scala
(100%)
rename client-spark/{spark-3 =>
spark-3-4}/src/main/scala/org/apache/spark/shuffle/celeborn/CelebornShuffleFallbackPolicyRunner.scala
(100%)
rename client-spark/{spark-3 =>
spark-3-4}/src/main/scala/org/apache/spark/shuffle/celeborn/CelebornShuffleHandle.scala
(100%)
rename client-spark/{spark-3 =>
spark-3-4}/src/main/scala/org/apache/spark/shuffle/celeborn/CelebornShuffleReader.scala
(100%)
rename client-spark/{spark-3 =>
spark-3-4}/src/test/java/org/apache/spark/shuffle/celeborn/CelebornShuffleWriterSuiteBase.java
(100%)
rename client-spark/{spark-3 =>
spark-3-4}/src/test/java/org/apache/spark/shuffle/celeborn/HashBasedShuffleWriterSuiteJ.java
(100%)
rename client-spark/{spark-3 =>
spark-3-4}/src/test/java/org/apache/spark/shuffle/celeborn/ShuffleManagerHook.java
(100%)
rename client-spark/{spark-3 =>
spark-3-4}/src/test/java/org/apache/spark/shuffle/celeborn/SortBasedShuffleWriterSuiteJ.java
(100%)
rename client-spark/{spark-3 =>
spark-3-4}/src/test/java/org/apache/spark/shuffle/celeborn/TestCelebornShuffleManager.java
(100%)
rename client-spark/{spark-3 => spark-3-4}/src/test/resources/log4j.properties
(100%)
rename client-spark/{spark-3 => spark-3-4}/src/test/resources/log4j2-test.xml
(100%)
rename client-spark/{spark-3 =>
spark-3-4}/src/test/scala/org/apache/spark/shuffle/celeborn/CelebornShuffleManagerSuite.scala
(100%)
copy client-spark/{spark-3-columnar-shuffle =>
spark-4-columnar-shuffle}/pom.xml (95%)
copy client-spark/{spark-3-columnar-shuffle =>
spark-4-columnar-shuffle}/src/main/java/org/apache/spark/shuffle/celeborn/ColumnarHashBasedShuffleWriter.java
(100%)
copy client-spark/{spark-3-columnar-shuffle =>
spark-4-columnar-shuffle}/src/main/scala/org/apache/spark/shuffle/celeborn/CelebornColumnarShuffleReader.scala
(100%)
copy client-spark/{spark-3.5-columnar-shuffle =>
spark-4-columnar-shuffle}/src/main/scala/org/apache/spark/sql/execution/columnar/CelebornColumnAccessor.scala
(100%)
copy client-spark/{spark-3.5-columnar-shuffle =>
spark-4-columnar-shuffle}/src/main/scala/org/apache/spark/sql/execution/columnar/CelebornColumnBuilder.scala
(99%)
copy client-spark/{spark-3-columnar-shuffle =>
spark-4-columnar-shuffle}/src/main/scala/org/apache/spark/sql/execution/columnar/CelebornColumnStats.scala
(100%)
copy client-spark/{spark-3.5-columnar-shuffle =>
spark-4-columnar-shuffle}/src/main/scala/org/apache/spark/sql/execution/columnar/CelebornColumnType.scala
(99%)
copy client-spark/{spark-3.5-columnar-shuffle =>
spark-4-columnar-shuffle}/src/main/scala/org/apache/spark/sql/execution/columnar/CelebornColumnarBatchBuilder.scala
(100%)
copy client-spark/{spark-3-columnar-shuffle =>
spark-4-columnar-shuffle}/src/main/scala/org/apache/spark/sql/execution/columnar/CelebornColumnarBatchCodeGenBuild.scala
(100%)
copy client-spark/{spark-3-columnar-shuffle =>
spark-4-columnar-shuffle}/src/main/scala/org/apache/spark/sql/execution/columnar/CelebornColumnarBatchSerializer.scala
(100%)
copy client-spark/{spark-3.5-columnar-shuffle =>
spark-4-columnar-shuffle}/src/main/scala/org/apache/spark/sql/execution/columnar/CelebornCompressibleColumnAccessor.scala
(100%)
copy client-spark/{spark-3.5-columnar-shuffle =>
spark-4-columnar-shuffle}/src/main/scala/org/apache/spark/sql/execution/columnar/CelebornCompressibleColumnBuilder.scala
(100%)
copy client-spark/{spark-3.5-columnar-shuffle =>
spark-4-columnar-shuffle}/src/main/scala/org/apache/spark/sql/execution/columnar/CelebornCompressionScheme.scala
(100%)
copy client-spark/{spark-3.5-columnar-shuffle =>
spark-4-columnar-shuffle}/src/main/scala/org/apache/spark/sql/execution/columnar/CelebornCompressionSchemes.scala
(100%)
copy client-spark/{spark-3-columnar-shuffle =>
spark-4-columnar-shuffle}/src/main/scala/org/apache/spark/sql/execution/columnar/CelebornNullableColumnAccessor.scala
(100%)
copy client-spark/{spark-3-columnar-shuffle =>
spark-4-columnar-shuffle}/src/main/scala/org/apache/spark/sql/execution/columnar/CelebornNullableColumnBuilder.scala
(100%)
copy client-spark/{spark-3-columnar-shuffle =>
spark-4-columnar-shuffle}/src/test/java/org/apache/spark/shuffle/celeborn/ColumnarHashBasedShuffleWriterSuiteJ.java
(100%)
copy {client-flink/flink-1.14 =>
client-spark/spark-4-columnar-shuffle}/src/test/resources/log4j2-test.xml (100%)
copy client-spark/{spark-3-columnar-shuffle =>
spark-4-columnar-shuffle}/src/test/scala/org/apache/spark/shuffle/celeborn/CelebornColumnarShuffleReaderSuite.scala
(100%)
copy {client-flink/flink-1.18-shaded => client-spark/spark-4-shaded}/pom.xml
(92%)
copy client-spark/{spark-3-shaded =>
spark-4-shaded}/src/main/resources/META-INF/LICENSE (100%)
copy client-spark/{spark-2-shaded =>
spark-4-shaded}/src/main/resources/META-INF/NOTICE (100%)
copy {client-flink/flink-1.14-shaded =>
client-spark/spark-4-shaded}/src/main/resources/META-INF/licenses/LICENSE-protobuf.txt
(100%)
