This is an automated email from the ASF dual-hosted git repository.
hongze pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 7ca509f9b [GLUTEN-7031] Move task lifecycle management / memory
consumer facilities to gluten-core (#7088)
7ca509f9b is described below
commit 7ca509f9b2c258473f92449cc02d6d4d71202cc9
Author: Hongze Zhang <[email protected]>
AuthorDate: Tue Sep 3 14:26:45 2024 +0800
[GLUTEN-7031] Move task lifecycle management / memory consumer facilities
to gluten-core (#7088)
---
.../org/apache/gluten/memory/CHThreadGroup.java | 4 ++--
.../GlutenClickHouseTPCDSMetricsSuite.scala | 2 +-
.../metrics/GlutenClickHouseTPCHMetricsSuite.scala | 2 +-
.../backendsapi/velox/VeloxTransformerApi.scala | 2 +-
.../backendsapi/velox/VeloxValidatorApi.scala | 2 +-
.../v2/ArrowCSVPartitionReaderFactory.scala | 3 ++-
.../gluten/datasource/v2/ArrowCSVTable.scala | 2 +-
.../gluten/execution/RowToVeloxColumnarExec.scala | 2 +-
.../expression/VeloxBloomFilterMightContain.scala | 2 +-
.../aggregate/VeloxBloomFilterAggregate.scala | 2 +-
.../spark/sql/execution/BroadcastUtils.scala | 2 +-
.../gluten/columnarbatch/ColumnarBatchTest.java | 2 +-
.../apache/gluten/utils/VeloxBloomFilterTest.java | 2 +-
.../gluten/vectorized/ArrowColumnVectorTest.java | 2 +-
cpp/core/CMakeLists.txt | 8 ++++----
.../VeloxCelebornColumnarBatchSerializer.scala | 2 +-
gluten-core/pom.xml | 22 ++++++++++++++++++++++
.../apache/gluten/memory/MemoryUsageRecorder.java | 0
.../gluten/memory/MemoryUsageStatsBuilder.java | 0
.../gluten/memory/SimpleMemoryUsageRecorder.java | 0
.../DynamicOffHeapSizingMemoryTarget.java | 0
.../gluten/memory/memtarget/KnownNameAndStats.java | 0
.../memory/memtarget/LoggingMemoryTarget.java | 0
.../gluten/memory/memtarget/MemoryTarget.java | 0
.../gluten/memory/memtarget/MemoryTargetUtil.java | 0
.../memory/memtarget/MemoryTargetVisitor.java | 0
.../gluten/memory/memtarget/MemoryTargets.java | 0
.../gluten/memory/memtarget/NoopMemoryTarget.java | 0
.../gluten/memory/memtarget/OverAcquire.java | 0
.../apache/gluten/memory/memtarget/Spiller.java | 0
.../apache/gluten/memory/memtarget/Spillers.java | 0
.../memory/memtarget/ThrowOnOomMemoryTarget.java | 2 +-
.../gluten/memory/memtarget/TreeMemoryTarget.java | 0
.../gluten/memory/memtarget/TreeMemoryTargets.java | 0
.../memtarget/spark/RegularMemoryConsumer.java | 2 +-
.../memory/memtarget/spark/TreeMemoryConsumer.java | 0
.../memtarget/spark/TreeMemoryConsumers.java | 0
.../org/apache/gluten/task}/TaskListener.scala | 2 +-
.../resources/org/apache/gluten/proto/config.proto | 0
.../resources/org/apache/gluten/proto/memory.proto | 0
.../org/apache/spark/memory/SparkMemoryUtil.scala | 2 +-
.../org/apache/spark/task}/TaskResource.scala | 2 +-
.../org/apache/spark/task}/TaskResources.scala | 20 +++++++++++---------
.../gluten/columnarbatch/IndicatorVector.java | 2 +-
.../gluten/columnarbatch/IndicatorVectorPool.java | 2 +-
.../memory/arrow/alloc/ArrowBufferAllocators.java | 4 ++--
.../memory/arrow/pool/ArrowNativeMemoryPool.java | 4 ++--
.../memory/listener/ReservationListeners.java | 2 +-
.../metrics/HashAggregateMetricsUpdater.scala | 2 +-
.../apache/gluten/metrics/JoinMetricsUpdater.scala | 2 +-
.../apache/gluten/metrics/SortMetricsUpdater.scala | 2 +-
.../scala/org/apache/gluten/runtime/Runtime.scala | 2 +-
.../scala/org/apache/gluten/runtime/Runtimes.scala | 2 +-
.../vectorized/ColumnarBatchSerializer.scala | 2 +-
.../sql/execution/ColumnarBuildSideRelation.scala | 2 +-
.../execution/MassiveMemoryAllocationSuite.scala | 2 +-
.../scala/org/apache/gluten/GlutenPlugin.scala | 5 +++--
.../gluten/planner/plan/GlutenPlanModel.scala | 3 ++-
.../apache/gluten/utils/iterator/IteratorsV1.scala | 2 +-
.../memtarget/spark/TreeMemoryConsumerTest.java | 2 +-
.../apache/gluten/utils/TaskResourceSuite.scala | 3 ++-
.../gluten/utils/iterator/IteratorSuite.scala | 2 +-
.../spark/utils/iterator/IteratorBenchmark.scala | 2 +-
63 files changed, 85 insertions(+), 57 deletions(-)
diff --git
a/backends-clickhouse/src/main/java/org/apache/gluten/memory/CHThreadGroup.java
b/backends-clickhouse/src/main/java/org/apache/gluten/memory/CHThreadGroup.java
index a06c552a9..119dc6189 100644
---
a/backends-clickhouse/src/main/java/org/apache/gluten/memory/CHThreadGroup.java
+++
b/backends-clickhouse/src/main/java/org/apache/gluten/memory/CHThreadGroup.java
@@ -17,8 +17,8 @@
package org.apache.gluten.memory;
import org.apache.spark.TaskContext;
-import org.apache.spark.util.TaskResource;
-import org.apache.spark.util.TaskResources;
+import org.apache.spark.task.TaskResource;
+import org.apache.spark.task.TaskResources;
public class CHThreadGroup implements TaskResource {
diff --git
a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/metrics/GlutenClickHouseTPCDSMetricsSuite.scala
b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/metrics/GlutenClickHouseTPCDSMetricsSuite.scala
index 79017eab0..3e3899b0e 100644
---
a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/metrics/GlutenClickHouseTPCDSMetricsSuite.scala
+++
b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/metrics/GlutenClickHouseTPCDSMetricsSuite.scala
@@ -23,7 +23,7 @@ import org.apache.gluten.vectorized.GeneralInIterator
import org.apache.spark.SparkConf
import org.apache.spark.sql.catalyst.expressions.Attribute
import org.apache.spark.sql.execution.InputIteratorTransformer
-import org.apache.spark.util.TaskResources
+import org.apache.spark.task.TaskResources
import scala.collection.JavaConverters._
diff --git
a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/metrics/GlutenClickHouseTPCHMetricsSuite.scala
b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/metrics/GlutenClickHouseTPCHMetricsSuite.scala
index 509967125..a5d81b781 100644
---
a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/metrics/GlutenClickHouseTPCHMetricsSuite.scala
+++
b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/metrics/GlutenClickHouseTPCHMetricsSuite.scala
@@ -23,7 +23,7 @@ import org.apache.gluten.vectorized.GeneralInIterator
import org.apache.spark.SparkConf
import org.apache.spark.sql.catalyst.expressions.Attribute
import org.apache.spark.sql.execution.InputIteratorTransformer
-import org.apache.spark.util.TaskResources
+import org.apache.spark.task.TaskResources
import scala.collection.JavaConverters._
diff --git
a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxTransformerApi.scala
b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxTransformerApi.scala
index 37d8acc7f..7cc4f5e81 100644
---
a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxTransformerApi.scala
+++
b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxTransformerApi.scala
@@ -28,7 +28,7 @@ import org.apache.spark.sql.catalyst.expressions.{Attribute,
Expression}
import org.apache.spark.sql.connector.read.InputPartition
import org.apache.spark.sql.execution.datasources.{HadoopFsRelation,
PartitionDirectory}
import org.apache.spark.sql.types._
-import org.apache.spark.util.TaskResources
+import org.apache.spark.task.TaskResources
import org.apache.spark.util.collection.BitSet
import com.google.protobuf.{Any, Message}
diff --git
a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxValidatorApi.scala
b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxValidatorApi.scala
index 131951a96..e15270f47 100644
---
a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxValidatorApi.scala
+++
b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxValidatorApi.scala
@@ -25,7 +25,7 @@ import org.apache.spark.sql.catalyst.expressions.Expression
import org.apache.spark.sql.catalyst.plans.physical.Partitioning
import org.apache.spark.sql.execution.SparkPlan
import org.apache.spark.sql.types._
-import org.apache.spark.util.TaskResources
+import org.apache.spark.task.TaskResources
class VeloxValidatorApi extends ValidatorApi {
diff --git
a/backends-velox/src/main/scala/org/apache/gluten/datasource/v2/ArrowCSVPartitionReaderFactory.scala
b/backends-velox/src/main/scala/org/apache/gluten/datasource/v2/ArrowCSVPartitionReaderFactory.scala
index 4af5022a6..c930cebeb 100644
---
a/backends-velox/src/main/scala/org/apache/gluten/datasource/v2/ArrowCSVPartitionReaderFactory.scala
+++
b/backends-velox/src/main/scala/org/apache/gluten/datasource/v2/ArrowCSVPartitionReaderFactory.scala
@@ -33,7 +33,8 @@ import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.sources.Filter
import org.apache.spark.sql.types.{StructField, StructType}
import org.apache.spark.sql.vectorized.ColumnarBatch
-import org.apache.spark.util.{SerializableConfiguration, TaskResources}
+import org.apache.spark.task.TaskResources
+import org.apache.spark.util.SerializableConfiguration
import org.apache.arrow.c.ArrowSchema
import org.apache.arrow.vector.types.pojo.Schema
diff --git
a/backends-velox/src/main/scala/org/apache/gluten/datasource/v2/ArrowCSVTable.scala
b/backends-velox/src/main/scala/org/apache/gluten/datasource/v2/ArrowCSVTable.scala
index 02485975e..3eaf4e35f 100644
---
a/backends-velox/src/main/scala/org/apache/gluten/datasource/v2/ArrowCSVTable.scala
+++
b/backends-velox/src/main/scala/org/apache/gluten/datasource/v2/ArrowCSVTable.scala
@@ -29,7 +29,7 @@ import org.apache.spark.sql.execution.datasources.FileFormat
import org.apache.spark.sql.execution.datasources.v2.FileTable
import org.apache.spark.sql.types.StructType
import org.apache.spark.sql.util.CaseInsensitiveStringMap
-import org.apache.spark.util.TaskResources
+import org.apache.spark.task.TaskResources
import org.apache.hadoop.fs.FileStatus
diff --git
a/backends-velox/src/main/scala/org/apache/gluten/execution/RowToVeloxColumnarExec.scala
b/backends-velox/src/main/scala/org/apache/gluten/execution/RowToVeloxColumnarExec.scala
index aa30cc80d..542c325f0 100644
---
a/backends-velox/src/main/scala/org/apache/gluten/execution/RowToVeloxColumnarExec.scala
+++
b/backends-velox/src/main/scala/org/apache/gluten/execution/RowToVeloxColumnarExec.scala
@@ -34,8 +34,8 @@ import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.types.StructType
import org.apache.spark.sql.utils.SparkArrowUtil
import org.apache.spark.sql.vectorized.ColumnarBatch
+import org.apache.spark.task.TaskResources
import org.apache.spark.unsafe.Platform
-import org.apache.spark.util.TaskResources
import org.apache.arrow.c.ArrowSchema
import org.apache.arrow.memory.ArrowBuf
diff --git
a/backends-velox/src/main/scala/org/apache/gluten/expression/VeloxBloomFilterMightContain.scala
b/backends-velox/src/main/scala/org/apache/gluten/expression/VeloxBloomFilterMightContain.scala
index 8c2a7e492..05e1e3eb4 100644
---
a/backends-velox/src/main/scala/org/apache/gluten/expression/VeloxBloomFilterMightContain.scala
+++
b/backends-velox/src/main/scala/org/apache/gluten/expression/VeloxBloomFilterMightContain.scala
@@ -25,7 +25,7 @@ import
org.apache.spark.sql.catalyst.expressions.{BinaryExpression, Expression}
import org.apache.spark.sql.catalyst.expressions.codegen._
import org.apache.spark.sql.catalyst.expressions.codegen.Block.BlockHelper
import org.apache.spark.sql.types.DataType
-import org.apache.spark.util.TaskResources
+import org.apache.spark.task.TaskResources
/**
* Velox's bloom-filter implementation uses different algorithms internally
comparing to vanilla
diff --git
a/backends-velox/src/main/scala/org/apache/gluten/expression/aggregate/VeloxBloomFilterAggregate.scala
b/backends-velox/src/main/scala/org/apache/gluten/expression/aggregate/VeloxBloomFilterAggregate.scala
index d22cc7023..976abb9e2 100644
---
a/backends-velox/src/main/scala/org/apache/gluten/expression/aggregate/VeloxBloomFilterAggregate.scala
+++
b/backends-velox/src/main/scala/org/apache/gluten/expression/aggregate/VeloxBloomFilterAggregate.scala
@@ -26,7 +26,7 @@ import
org.apache.spark.sql.catalyst.expressions.aggregate.TypedImperativeAggreg
import org.apache.spark.sql.catalyst.trees.TernaryLike
import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.types.DataType
-import org.apache.spark.util.TaskResources
+import org.apache.spark.task.TaskResources
import org.apache.spark.util.sketch.BloomFilter
/**
diff --git
a/backends-velox/src/main/scala/org/apache/spark/sql/execution/BroadcastUtils.scala
b/backends-velox/src/main/scala/org/apache/spark/sql/execution/BroadcastUtils.scala
index 29a12f532..5bc6b7c56 100644
---
a/backends-velox/src/main/scala/org/apache/spark/sql/execution/BroadcastUtils.scala
+++
b/backends-velox/src/main/scala/org/apache/spark/sql/execution/BroadcastUtils.scala
@@ -29,7 +29,7 @@ import
org.apache.spark.sql.catalyst.plans.physical.{BroadcastMode, BroadcastPar
import org.apache.spark.sql.execution.joins.{HashedRelation,
HashedRelationBroadcastMode, LongHashedRelation}
import org.apache.spark.sql.types.StructType
import org.apache.spark.sql.vectorized.ColumnarBatch
-import org.apache.spark.util.TaskResources
+import org.apache.spark.task.TaskResources
import scala.collection.mutable.ArrayBuffer;
diff --git
a/backends-velox/src/test/java/org/apache/gluten/columnarbatch/ColumnarBatchTest.java
b/backends-velox/src/test/java/org/apache/gluten/columnarbatch/ColumnarBatchTest.java
index 54994ccd4..819d35a10 100644
---
a/backends-velox/src/test/java/org/apache/gluten/columnarbatch/ColumnarBatchTest.java
+++
b/backends-velox/src/test/java/org/apache/gluten/columnarbatch/ColumnarBatchTest.java
@@ -24,7 +24,7 @@ import org.apache.gluten.vectorized.ArrowWritableColumnVector;
import org.apache.spark.sql.types.DataTypes;
import org.apache.spark.sql.types.StructType;
import org.apache.spark.sql.vectorized.ColumnarBatch;
-import org.apache.spark.util.TaskResources$;
+import org.apache.spark.task.TaskResources$;
import org.junit.Assert;
import org.junit.Test;
diff --git
a/backends-velox/src/test/java/org/apache/gluten/utils/VeloxBloomFilterTest.java
b/backends-velox/src/test/java/org/apache/gluten/utils/VeloxBloomFilterTest.java
index cf568b166..86b14fa46 100644
---
a/backends-velox/src/test/java/org/apache/gluten/utils/VeloxBloomFilterTest.java
+++
b/backends-velox/src/test/java/org/apache/gluten/utils/VeloxBloomFilterTest.java
@@ -18,7 +18,7 @@ package org.apache.gluten.utils;
import org.apache.gluten.test.VeloxBackendTestBase;
-import org.apache.spark.util.TaskResources$;
+import org.apache.spark.task.TaskResources$;
import org.apache.spark.util.sketch.BloomFilter;
import org.apache.spark.util.sketch.IncompatibleMergeException;
import org.junit.Assert;
diff --git
a/backends-velox/src/test/java/org/apache/gluten/vectorized/ArrowColumnVectorTest.java
b/backends-velox/src/test/java/org/apache/gluten/vectorized/ArrowColumnVectorTest.java
index 11330544d..c7c13450c 100644
---
a/backends-velox/src/test/java/org/apache/gluten/vectorized/ArrowColumnVectorTest.java
+++
b/backends-velox/src/test/java/org/apache/gluten/vectorized/ArrowColumnVectorTest.java
@@ -19,7 +19,7 @@ package org.apache.gluten.vectorized;
import org.apache.spark.sql.execution.vectorized.MutableColumnarRow;
import org.apache.spark.sql.types.Decimal;
import org.apache.spark.sql.types.StructType;
-import org.apache.spark.util.TaskResources$;
+import org.apache.spark.task.TaskResources$;
import org.junit.Assert;
import org.junit.Test;
diff --git a/cpp/core/CMakeLists.txt b/cpp/core/CMakeLists.txt
index 9311b71ed..4c84d15f3 100644
--- a/cpp/core/CMakeLists.txt
+++ b/cpp/core/CMakeLists.txt
@@ -38,14 +38,14 @@ endif()
set(CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/CMake" ${CMAKE_MODULE_PATH})
+set(GLUTEN_PROTO_SRC_DIR
+ ${GLUTEN_HOME}/gluten-core/src/main/resources/org/apache/gluten/proto)
+message(STATUS "Set Gluten Proto Directory in ${GLUTEN_PROTO_SRC_DIR}")
+
set(SUBSTRAIT_PROTO_SRC_DIR
${GLUTEN_HOME}/gluten-substrait/src/main/resources/substrait/proto)
message(STATUS "Set Substrait Proto Directory in ${SUBSTRAIT_PROTO_SRC_DIR}")
-set(GLUTEN_PROTO_SRC_DIR
- ${GLUTEN_HOME}/gluten-substrait/src/main/resources/org/apache/gluten/proto)
-message(STATUS "Set Gluten Proto Directory in ${GLUTEN_PROTO_SRC_DIR}")
-
find_program(CCACHE_FOUND ccache)
if(CCACHE_FOUND)
set_property(GLOBAL PROPERTY RULE_LAUNCH_COMPILE ccache)
diff --git
a/gluten-celeborn/velox/src/main/scala/org/apache/spark/shuffle/VeloxCelebornColumnarBatchSerializer.scala
b/gluten-celeborn/velox/src/main/scala/org/apache/spark/shuffle/VeloxCelebornColumnarBatchSerializer.scala
index 696a3c343..5355baed5 100644
---
a/gluten-celeborn/velox/src/main/scala/org/apache/spark/shuffle/VeloxCelebornColumnarBatchSerializer.scala
+++
b/gluten-celeborn/velox/src/main/scala/org/apache/spark/shuffle/VeloxCelebornColumnarBatchSerializer.scala
@@ -32,7 +32,7 @@ import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.types.StructType
import org.apache.spark.sql.utils.SparkSchemaUtil
import org.apache.spark.sql.vectorized.ColumnarBatch
-import org.apache.spark.util.{TaskResource, TaskResources}
+import org.apache.spark.task.{TaskResource, TaskResources}
import org.apache.arrow.c.ArrowSchema
import org.apache.arrow.memory.BufferAllocator
diff --git a/gluten-core/pom.xml b/gluten-core/pom.xml
index 08f5de580..448bc9ee2 100644
--- a/gluten-core/pom.xml
+++ b/gluten-core/pom.xml
@@ -128,6 +128,28 @@
<outputDirectory>target/scala-${scala.binary.version}/classes</outputDirectory>
<testOutputDirectory>target/scala-${scala.binary.version}/test-classes</testOutputDirectory>
<plugins>
+ <!-- compile proto buffer files using copied protoc binary -->
+ <plugin>
+ <groupId>org.xolstice.maven.plugins</groupId>
+ <artifactId>protobuf-maven-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>compile-gluten-proto</id>
+ <phase>generate-sources</phase>
+ <goals>
+ <goal>compile</goal>
+ <goal>test-compile</goal>
+ </goals>
+ <configuration>
+ <protocArtifact>
+
com.google.protobuf:protoc:${protobuf.version}:exe:${os.detected.classifier}
+ </protocArtifact>
+
<protoSourceRoot>src/main/resources/org/apache/gluten/proto</protoSourceRoot>
+ <clearOutputDirectory>false</clearOutputDirectory>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-resources-plugin</artifactId>
diff --git
a/gluten-substrait/src/main/java/org/apache/gluten/memory/MemoryUsageRecorder.java
b/gluten-core/src/main/java/org/apache/gluten/memory/MemoryUsageRecorder.java
similarity index 100%
rename from
gluten-substrait/src/main/java/org/apache/gluten/memory/MemoryUsageRecorder.java
rename to
gluten-core/src/main/java/org/apache/gluten/memory/MemoryUsageRecorder.java
diff --git
a/gluten-substrait/src/main/java/org/apache/gluten/memory/MemoryUsageStatsBuilder.java
b/gluten-core/src/main/java/org/apache/gluten/memory/MemoryUsageStatsBuilder.java
similarity index 100%
rename from
gluten-substrait/src/main/java/org/apache/gluten/memory/MemoryUsageStatsBuilder.java
rename to
gluten-core/src/main/java/org/apache/gluten/memory/MemoryUsageStatsBuilder.java
diff --git
a/gluten-substrait/src/main/java/org/apache/gluten/memory/SimpleMemoryUsageRecorder.java
b/gluten-core/src/main/java/org/apache/gluten/memory/SimpleMemoryUsageRecorder.java
similarity index 100%
rename from
gluten-substrait/src/main/java/org/apache/gluten/memory/SimpleMemoryUsageRecorder.java
rename to
gluten-core/src/main/java/org/apache/gluten/memory/SimpleMemoryUsageRecorder.java
diff --git
a/gluten-substrait/src/main/java/org/apache/gluten/memory/memtarget/DynamicOffHeapSizingMemoryTarget.java
b/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/DynamicOffHeapSizingMemoryTarget.java
similarity index 100%
rename from
gluten-substrait/src/main/java/org/apache/gluten/memory/memtarget/DynamicOffHeapSizingMemoryTarget.java
rename to
gluten-core/src/main/java/org/apache/gluten/memory/memtarget/DynamicOffHeapSizingMemoryTarget.java
diff --git
a/gluten-substrait/src/main/java/org/apache/gluten/memory/memtarget/KnownNameAndStats.java
b/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/KnownNameAndStats.java
similarity index 100%
rename from
gluten-substrait/src/main/java/org/apache/gluten/memory/memtarget/KnownNameAndStats.java
rename to
gluten-core/src/main/java/org/apache/gluten/memory/memtarget/KnownNameAndStats.java
diff --git
a/gluten-substrait/src/main/java/org/apache/gluten/memory/memtarget/LoggingMemoryTarget.java
b/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/LoggingMemoryTarget.java
similarity index 100%
rename from
gluten-substrait/src/main/java/org/apache/gluten/memory/memtarget/LoggingMemoryTarget.java
rename to
gluten-core/src/main/java/org/apache/gluten/memory/memtarget/LoggingMemoryTarget.java
diff --git
a/gluten-substrait/src/main/java/org/apache/gluten/memory/memtarget/MemoryTarget.java
b/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/MemoryTarget.java
similarity index 100%
rename from
gluten-substrait/src/main/java/org/apache/gluten/memory/memtarget/MemoryTarget.java
rename to
gluten-core/src/main/java/org/apache/gluten/memory/memtarget/MemoryTarget.java
diff --git
a/gluten-substrait/src/main/java/org/apache/gluten/memory/memtarget/MemoryTargetUtil.java
b/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/MemoryTargetUtil.java
similarity index 100%
rename from
gluten-substrait/src/main/java/org/apache/gluten/memory/memtarget/MemoryTargetUtil.java
rename to
gluten-core/src/main/java/org/apache/gluten/memory/memtarget/MemoryTargetUtil.java
diff --git
a/gluten-substrait/src/main/java/org/apache/gluten/memory/memtarget/MemoryTargetVisitor.java
b/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/MemoryTargetVisitor.java
similarity index 100%
rename from
gluten-substrait/src/main/java/org/apache/gluten/memory/memtarget/MemoryTargetVisitor.java
rename to
gluten-core/src/main/java/org/apache/gluten/memory/memtarget/MemoryTargetVisitor.java
diff --git
a/gluten-substrait/src/main/java/org/apache/gluten/memory/memtarget/MemoryTargets.java
b/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/MemoryTargets.java
similarity index 100%
rename from
gluten-substrait/src/main/java/org/apache/gluten/memory/memtarget/MemoryTargets.java
rename to
gluten-core/src/main/java/org/apache/gluten/memory/memtarget/MemoryTargets.java
diff --git
a/gluten-substrait/src/main/java/org/apache/gluten/memory/memtarget/NoopMemoryTarget.java
b/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/NoopMemoryTarget.java
similarity index 100%
rename from
gluten-substrait/src/main/java/org/apache/gluten/memory/memtarget/NoopMemoryTarget.java
rename to
gluten-core/src/main/java/org/apache/gluten/memory/memtarget/NoopMemoryTarget.java
diff --git
a/gluten-substrait/src/main/java/org/apache/gluten/memory/memtarget/OverAcquire.java
b/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/OverAcquire.java
similarity index 100%
rename from
gluten-substrait/src/main/java/org/apache/gluten/memory/memtarget/OverAcquire.java
rename to
gluten-core/src/main/java/org/apache/gluten/memory/memtarget/OverAcquire.java
diff --git
a/gluten-substrait/src/main/java/org/apache/gluten/memory/memtarget/Spiller.java
b/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/Spiller.java
similarity index 100%
rename from
gluten-substrait/src/main/java/org/apache/gluten/memory/memtarget/Spiller.java
rename to
gluten-core/src/main/java/org/apache/gluten/memory/memtarget/Spiller.java
diff --git
a/gluten-substrait/src/main/java/org/apache/gluten/memory/memtarget/Spillers.java
b/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/Spillers.java
similarity index 100%
rename from
gluten-substrait/src/main/java/org/apache/gluten/memory/memtarget/Spillers.java
rename to
gluten-core/src/main/java/org/apache/gluten/memory/memtarget/Spillers.java
diff --git
a/gluten-substrait/src/main/java/org/apache/gluten/memory/memtarget/ThrowOnOomMemoryTarget.java
b/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/ThrowOnOomMemoryTarget.java
similarity index 99%
rename from
gluten-substrait/src/main/java/org/apache/gluten/memory/memtarget/ThrowOnOomMemoryTarget.java
rename to
gluten-core/src/main/java/org/apache/gluten/memory/memtarget/ThrowOnOomMemoryTarget.java
index 74e0cbb87..67edb713a 100644
---
a/gluten-substrait/src/main/java/org/apache/gluten/memory/memtarget/ThrowOnOomMemoryTarget.java
+++
b/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/ThrowOnOomMemoryTarget.java
@@ -20,7 +20,7 @@ import org.apache.gluten.GlutenConfig$;
import org.apache.spark.memory.SparkMemoryUtil;
import org.apache.spark.sql.internal.SQLConf;
-import org.apache.spark.util.TaskResources;
+import org.apache.spark.task.TaskResources;
import org.apache.spark.util.Utils;
public class ThrowOnOomMemoryTarget implements MemoryTarget {
diff --git
a/gluten-substrait/src/main/java/org/apache/gluten/memory/memtarget/TreeMemoryTarget.java
b/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/TreeMemoryTarget.java
similarity index 100%
rename from
gluten-substrait/src/main/java/org/apache/gluten/memory/memtarget/TreeMemoryTarget.java
rename to
gluten-core/src/main/java/org/apache/gluten/memory/memtarget/TreeMemoryTarget.java
diff --git
a/gluten-substrait/src/main/java/org/apache/gluten/memory/memtarget/TreeMemoryTargets.java
b/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/TreeMemoryTargets.java
similarity index 100%
rename from
gluten-substrait/src/main/java/org/apache/gluten/memory/memtarget/TreeMemoryTargets.java
rename to
gluten-core/src/main/java/org/apache/gluten/memory/memtarget/TreeMemoryTargets.java
diff --git
a/gluten-substrait/src/main/java/org/apache/gluten/memory/memtarget/spark/RegularMemoryConsumer.java
b/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/spark/RegularMemoryConsumer.java
similarity index 99%
rename from
gluten-substrait/src/main/java/org/apache/gluten/memory/memtarget/spark/RegularMemoryConsumer.java
rename to
gluten-core/src/main/java/org/apache/gluten/memory/memtarget/spark/RegularMemoryConsumer.java
index 76aa63aeb..054b9ef10 100644
---
a/gluten-substrait/src/main/java/org/apache/gluten/memory/memtarget/spark/RegularMemoryConsumer.java
+++
b/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/spark/RegularMemoryConsumer.java
@@ -30,7 +30,7 @@ import com.google.common.base.Preconditions;
import org.apache.spark.memory.MemoryConsumer;
import org.apache.spark.memory.MemoryMode;
import org.apache.spark.memory.TaskMemoryManager;
-import org.apache.spark.util.TaskResources;
+import org.apache.spark.task.TaskResources;
import java.util.Map;
import java.util.stream.Collectors;
diff --git
a/gluten-substrait/src/main/java/org/apache/gluten/memory/memtarget/spark/TreeMemoryConsumer.java
b/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/spark/TreeMemoryConsumer.java
similarity index 100%
rename from
gluten-substrait/src/main/java/org/apache/gluten/memory/memtarget/spark/TreeMemoryConsumer.java
rename to
gluten-core/src/main/java/org/apache/gluten/memory/memtarget/spark/TreeMemoryConsumer.java
diff --git
a/gluten-substrait/src/main/java/org/apache/gluten/memory/memtarget/spark/TreeMemoryConsumers.java
b/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/spark/TreeMemoryConsumers.java
similarity index 100%
rename from
gluten-substrait/src/main/java/org/apache/gluten/memory/memtarget/spark/TreeMemoryConsumers.java
rename to
gluten-core/src/main/java/org/apache/gluten/memory/memtarget/spark/TreeMemoryConsumers.java
diff --git
a/gluten-substrait/src/main/scala/org/apache/gluten/utils/TaskListener.scala
b/gluten-core/src/main/java/org/apache/gluten/task/TaskListener.scala
similarity index 96%
rename from
gluten-substrait/src/main/scala/org/apache/gluten/utils/TaskListener.scala
rename to gluten-core/src/main/java/org/apache/gluten/task/TaskListener.scala
index 3ffb3206b..d99c7e885 100644
--- a/gluten-substrait/src/main/scala/org/apache/gluten/utils/TaskListener.scala
+++ b/gluten-core/src/main/java/org/apache/gluten/task/TaskListener.scala
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.gluten.utils
+package org.apache.gluten.task
import org.apache.spark.TaskFailedReason
diff --git
a/gluten-substrait/src/main/resources/org/apache/gluten/proto/config.proto
b/gluten-core/src/main/resources/org/apache/gluten/proto/config.proto
similarity index 100%
rename from
gluten-substrait/src/main/resources/org/apache/gluten/proto/config.proto
rename to gluten-core/src/main/resources/org/apache/gluten/proto/config.proto
diff --git
a/gluten-substrait/src/main/resources/org/apache/gluten/proto/memory.proto
b/gluten-core/src/main/resources/org/apache/gluten/proto/memory.proto
similarity index 100%
rename from
gluten-substrait/src/main/resources/org/apache/gluten/proto/memory.proto
rename to gluten-core/src/main/resources/org/apache/gluten/proto/memory.proto
diff --git
a/gluten-substrait/src/main/scala/org/apache/spark/memory/SparkMemoryUtil.scala
b/gluten-core/src/main/scala/org/apache/spark/memory/SparkMemoryUtil.scala
similarity index 97%
rename from
gluten-substrait/src/main/scala/org/apache/spark/memory/SparkMemoryUtil.scala
rename to
gluten-core/src/main/scala/org/apache/spark/memory/SparkMemoryUtil.scala
index 6fd2d7a3e..d221fafce 100644
---
a/gluten-substrait/src/main/scala/org/apache/spark/memory/SparkMemoryUtil.scala
+++ b/gluten-core/src/main/scala/org/apache/spark/memory/SparkMemoryUtil.scala
@@ -16,7 +16,7 @@
*/
package org.apache.spark.memory
-import org.apache.gluten.memory.memtarget.{DynamicOffHeapSizingMemoryTarget,
KnownNameAndStats, LoggingMemoryTarget, MemoryTarget, MemoryTargetVisitor,
NoopMemoryTarget, OverAcquire, ThrowOnOomMemoryTarget, TreeMemoryTargets}
+import org.apache.gluten.memory.memtarget._
import org.apache.gluten.memory.memtarget.spark.{RegularMemoryConsumer,
TreeMemoryConsumer}
import org.apache.gluten.proto.MemoryUsageStats
diff --git
a/gluten-substrait/src/main/scala/org/apache/spark/util/TaskResource.scala
b/gluten-core/src/main/scala/org/apache/spark/task/TaskResource.scala
similarity index 97%
rename from
gluten-substrait/src/main/scala/org/apache/spark/util/TaskResource.scala
rename to gluten-core/src/main/scala/org/apache/spark/task/TaskResource.scala
index 8dfa2110b..ca9bbd9dd 100644
--- a/gluten-substrait/src/main/scala/org/apache/spark/util/TaskResource.scala
+++ b/gluten-core/src/main/scala/org/apache/spark/task/TaskResource.scala
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.spark.util
+package org.apache.spark.task
/**
* Manages the lifecycle for a specific type of memory resource managed by
Spark. See also
diff --git
a/gluten-substrait/src/main/scala/org/apache/spark/util/TaskResources.scala
b/gluten-core/src/main/scala/org/apache/spark/task/TaskResources.scala
similarity index 95%
rename from
gluten-substrait/src/main/scala/org/apache/spark/util/TaskResources.scala
rename to gluten-core/src/main/scala/org/apache/spark/task/TaskResources.scala
index 2ab2a41a4..b061aa332 100644
--- a/gluten-substrait/src/main/scala/org/apache/spark/util/TaskResources.scala
+++ b/gluten-core/src/main/scala/org/apache/spark/task/TaskResources.scala
@@ -14,15 +14,17 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.spark.util
+package org.apache.spark.task
+
+import org.apache.gluten.task.TaskListener
import org.apache.spark.{TaskContext, TaskFailedReason, TaskKilledException,
UnknownReason}
import org.apache.spark.internal.Logging
import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.util.{TaskCompletionListener, TaskFailureListener}
import _root_.org.apache.gluten.memory.SimpleMemoryUsageRecorder
import _root_.org.apache.gluten.sql.shims.SparkShimLoader
-import _root_.org.apache.gluten.utils.TaskListener
import java.util
import java.util.{Collections, Properties, UUID}
@@ -288,7 +290,7 @@ class TaskResourceRegistry extends Logging {
}
/** Release all managed resources according to priority and reversed order */
- private[util] def releaseAll(): Unit = lock {
+ private[task] def releaseAll(): Unit = lock {
val table = new util.ArrayList(priorityToResourcesMapping.entrySet())
Collections.sort(
table,
@@ -310,7 +312,7 @@ class TaskResourceRegistry extends Logging {
}
/** Release single resource by ID */
- private[util] def releaseResource(id: String): Unit = lock {
+ private[task] def releaseResource(id: String): Unit = lock {
if (!resources.containsKey(id)) {
throw new IllegalArgumentException(
String.format("TaskResource with ID %s is not registered", id))
@@ -328,7 +330,7 @@ class TaskResourceRegistry extends Logging {
resources.remove(id)
}
- private[util] def addResourceIfNotRegistered[T <: TaskResource](id: String,
factory: () => T): T =
+ private[task] def addResourceIfNotRegistered[T <: TaskResource](id: String,
factory: () => T): T =
lock {
if (resources.containsKey(id)) {
return resources.get(id).asInstanceOf[T]
@@ -338,7 +340,7 @@ class TaskResourceRegistry extends Logging {
resource
}
- private[util] def addResource[T <: TaskResource](id: String, resource: T): T
= lock {
+ private[task] def addResource[T <: TaskResource](id: String, resource: T): T
= lock {
if (resources.containsKey(id)) {
throw new IllegalArgumentException(
String.format("TaskResource with ID %s is already registered", id))
@@ -347,11 +349,11 @@ class TaskResourceRegistry extends Logging {
resource
}
- private[util] def isResourceRegistered(id: String): Boolean = lock {
+ private[task] def isResourceRegistered(id: String): Boolean = lock {
resources.containsKey(id)
}
- private[util] def getResource[T <: TaskResource](id: String): T = lock {
+ private[task] def getResource[T <: TaskResource](id: String): T = lock {
if (!resources.containsKey(id)) {
throw new IllegalArgumentException(
String.format("TaskResource with ID %s is not registered", id))
@@ -359,7 +361,7 @@ class TaskResourceRegistry extends Logging {
resources.get(id).asInstanceOf[T]
}
- private[util] def getSharedUsage(): SimpleMemoryUsageRecorder = lock {
+ private[task] def getSharedUsage(): SimpleMemoryUsageRecorder = lock {
sharedUsage
}
}
diff --git
a/gluten-data/src/main/java/org/apache/gluten/columnarbatch/IndicatorVector.java
b/gluten-data/src/main/java/org/apache/gluten/columnarbatch/IndicatorVector.java
index 7fe87e95f..3f09a3619 100644
---
a/gluten-data/src/main/java/org/apache/gluten/columnarbatch/IndicatorVector.java
+++
b/gluten-data/src/main/java/org/apache/gluten/columnarbatch/IndicatorVector.java
@@ -16,7 +16,7 @@
*/
package org.apache.gluten.columnarbatch;
-import org.apache.spark.util.TaskResources;
+import org.apache.spark.task.TaskResources;
import java.util.concurrent.atomic.AtomicLong;
diff --git
a/gluten-data/src/main/java/org/apache/gluten/columnarbatch/IndicatorVectorPool.java
b/gluten-data/src/main/java/org/apache/gluten/columnarbatch/IndicatorVectorPool.java
index 6e46742b5..c122cc1cc 100644
---
a/gluten-data/src/main/java/org/apache/gluten/columnarbatch/IndicatorVectorPool.java
+++
b/gluten-data/src/main/java/org/apache/gluten/columnarbatch/IndicatorVectorPool.java
@@ -16,7 +16,7 @@
*/
package org.apache.gluten.columnarbatch;
-import org.apache.spark.util.TaskResource;
+import org.apache.spark.task.TaskResource;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git
a/gluten-data/src/main/java/org/apache/gluten/memory/arrow/alloc/ArrowBufferAllocators.java
b/gluten-data/src/main/java/org/apache/gluten/memory/arrow/alloc/ArrowBufferAllocators.java
index f51852ab8..4a57a0349 100644
---
a/gluten-data/src/main/java/org/apache/gluten/memory/arrow/alloc/ArrowBufferAllocators.java
+++
b/gluten-data/src/main/java/org/apache/gluten/memory/arrow/alloc/ArrowBufferAllocators.java
@@ -23,8 +23,8 @@ import org.apache.arrow.memory.AllocationListener;
import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.memory.RootAllocator;
import org.apache.spark.memory.TaskMemoryManager;
-import org.apache.spark.util.TaskResource;
-import org.apache.spark.util.TaskResources;
+import org.apache.spark.task.TaskResource;
+import org.apache.spark.task.TaskResources;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git
a/gluten-data/src/main/java/org/apache/gluten/memory/arrow/pool/ArrowNativeMemoryPool.java
b/gluten-data/src/main/java/org/apache/gluten/memory/arrow/pool/ArrowNativeMemoryPool.java
index 04a6e0002..a1f07c949 100644
---
a/gluten-data/src/main/java/org/apache/gluten/memory/arrow/pool/ArrowNativeMemoryPool.java
+++
b/gluten-data/src/main/java/org/apache/gluten/memory/arrow/pool/ArrowNativeMemoryPool.java
@@ -17,8 +17,8 @@
package org.apache.gluten.memory.arrow.pool;
import org.apache.arrow.dataset.jni.NativeMemoryPool;
-import org.apache.spark.util.TaskResource;
-import org.apache.spark.util.TaskResources;
+import org.apache.spark.task.TaskResource;
+import org.apache.spark.task.TaskResources;
import org.apache.spark.util.Utils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git
a/gluten-data/src/main/java/org/apache/gluten/memory/listener/ReservationListeners.java
b/gluten-data/src/main/java/org/apache/gluten/memory/listener/ReservationListeners.java
index db5ac8426..b221db13e 100644
---
a/gluten-data/src/main/java/org/apache/gluten/memory/listener/ReservationListeners.java
+++
b/gluten-data/src/main/java/org/apache/gluten/memory/listener/ReservationListeners.java
@@ -22,7 +22,7 @@ import org.apache.gluten.memory.SimpleMemoryUsageRecorder;
import org.apache.gluten.memory.memtarget.*;
import org.apache.spark.memory.TaskMemoryManager;
-import org.apache.spark.util.TaskResources;
+import org.apache.spark.task.TaskResources;
import java.util.Collections;
import java.util.Map;
diff --git
a/gluten-data/src/main/scala/org/apache/gluten/metrics/HashAggregateMetricsUpdater.scala
b/gluten-data/src/main/scala/org/apache/gluten/metrics/HashAggregateMetricsUpdater.scala
index 5053cc2ba..1bf422b0a 100644
---
a/gluten-data/src/main/scala/org/apache/gluten/metrics/HashAggregateMetricsUpdater.scala
+++
b/gluten-data/src/main/scala/org/apache/gluten/metrics/HashAggregateMetricsUpdater.scala
@@ -20,7 +20,7 @@ import org.apache.gluten.substrait.AggregationParams
import org.apache.spark.sql.execution.metric.SQLMetric
import org.apache.spark.sql.utils.SparkMetricsUtil
-import org.apache.spark.util.TaskResources
+import org.apache.spark.task.TaskResources
trait HashAggregateMetricsUpdater extends MetricsUpdater {
def updateAggregationMetrics(
diff --git
a/gluten-data/src/main/scala/org/apache/gluten/metrics/JoinMetricsUpdater.scala
b/gluten-data/src/main/scala/org/apache/gluten/metrics/JoinMetricsUpdater.scala
index fe1fa2ad6..9b8f03dd7 100644
---
a/gluten-data/src/main/scala/org/apache/gluten/metrics/JoinMetricsUpdater.scala
+++
b/gluten-data/src/main/scala/org/apache/gluten/metrics/JoinMetricsUpdater.scala
@@ -21,7 +21,7 @@ import org.apache.gluten.substrait.JoinParams
import org.apache.spark.sql.execution.metric.SQLMetric
import org.apache.spark.sql.utils.SparkMetricsUtil
-import org.apache.spark.util.TaskResources
+import org.apache.spark.task.TaskResources
import java.util
diff --git
a/gluten-data/src/main/scala/org/apache/gluten/metrics/SortMetricsUpdater.scala
b/gluten-data/src/main/scala/org/apache/gluten/metrics/SortMetricsUpdater.scala
index 0ba48c62c..cb01ae151 100644
---
a/gluten-data/src/main/scala/org/apache/gluten/metrics/SortMetricsUpdater.scala
+++
b/gluten-data/src/main/scala/org/apache/gluten/metrics/SortMetricsUpdater.scala
@@ -18,7 +18,7 @@ package org.apache.gluten.metrics
import org.apache.spark.sql.execution.metric.SQLMetric
import org.apache.spark.sql.utils.SparkMetricsUtil
-import org.apache.spark.util.TaskResources
+import org.apache.spark.task.TaskResources
class SortMetricsUpdater(val metrics: Map[String, SQLMetric]) extends
MetricsUpdater {
diff --git a/gluten-data/src/main/scala/org/apache/gluten/runtime/Runtime.scala
b/gluten-data/src/main/scala/org/apache/gluten/runtime/Runtime.scala
index 63db3e58e..b18d4044d 100644
--- a/gluten-data/src/main/scala/org/apache/gluten/runtime/Runtime.scala
+++ b/gluten-data/src/main/scala/org/apache/gluten/runtime/Runtime.scala
@@ -27,7 +27,7 @@ import org.apache.gluten.proto.MemoryUsageStats
import org.apache.spark.memory.SparkMemoryUtil
import org.apache.spark.sql.internal.{GlutenConfigUtil, SQLConf}
-import org.apache.spark.util.TaskResource
+import org.apache.spark.task.TaskResource
import org.slf4j.LoggerFactory
diff --git
a/gluten-data/src/main/scala/org/apache/gluten/runtime/Runtimes.scala
b/gluten-data/src/main/scala/org/apache/gluten/runtime/Runtimes.scala
index 6d5e11afe..bfb2465b1 100644
--- a/gluten-data/src/main/scala/org/apache/gluten/runtime/Runtimes.scala
+++ b/gluten-data/src/main/scala/org/apache/gluten/runtime/Runtimes.scala
@@ -16,7 +16,7 @@
*/
package org.apache.gluten.runtime
-import org.apache.spark.util.{TaskResource, TaskResources}
+import org.apache.spark.task.{TaskResource, TaskResources}
object Runtimes {
diff --git
a/gluten-data/src/main/scala/org/apache/gluten/vectorized/ColumnarBatchSerializer.scala
b/gluten-data/src/main/scala/org/apache/gluten/vectorized/ColumnarBatchSerializer.scala
index 0bd78cb92..c9f0d06f5 100644
---
a/gluten-data/src/main/scala/org/apache/gluten/vectorized/ColumnarBatchSerializer.scala
+++
b/gluten-data/src/main/scala/org/apache/gluten/vectorized/ColumnarBatchSerializer.scala
@@ -30,7 +30,7 @@ import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.types.StructType
import org.apache.spark.sql.utils.SparkSchemaUtil
import org.apache.spark.sql.vectorized.ColumnarBatch
-import org.apache.spark.util.{TaskResource, TaskResources}
+import org.apache.spark.task.{TaskResource, TaskResources}
import org.apache.arrow.c.ArrowSchema
import org.apache.arrow.memory.BufferAllocator
diff --git
a/gluten-data/src/main/scala/org/apache/spark/sql/execution/ColumnarBuildSideRelation.scala
b/gluten-data/src/main/scala/org/apache/spark/sql/execution/ColumnarBuildSideRelation.scala
index cb65dbca4..09805cc2f 100644
---
a/gluten-data/src/main/scala/org/apache/spark/sql/execution/ColumnarBuildSideRelation.scala
+++
b/gluten-data/src/main/scala/org/apache/spark/sql/execution/ColumnarBuildSideRelation.scala
@@ -30,7 +30,7 @@ import org.apache.spark.sql.execution.joins.BuildSideRelation
import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.utils.SparkArrowUtil
import org.apache.spark.sql.vectorized.ColumnarBatch
-import org.apache.spark.util.TaskResources
+import org.apache.spark.task.TaskResources
import org.apache.arrow.c.ArrowSchema
diff --git
a/gluten-data/src/test/scala/org/apache/gluten/execution/MassiveMemoryAllocationSuite.scala
b/gluten-data/src/test/scala/org/apache/gluten/execution/MassiveMemoryAllocationSuite.scala
index ebfa0e612..f53c99e5e 100644
---
a/gluten-data/src/test/scala/org/apache/gluten/execution/MassiveMemoryAllocationSuite.scala
+++
b/gluten-data/src/test/scala/org/apache/gluten/execution/MassiveMemoryAllocationSuite.scala
@@ -22,7 +22,7 @@ import org.apache.gluten.memory.memtarget.{MemoryTarget,
Spiller, Spillers}
import org.apache.spark.SparkFunSuite
import org.apache.spark.sql.test.SharedSparkSession
-import org.apache.spark.util.TaskResources
+import org.apache.spark.task.TaskResources
import java.util.concurrent.{Callable, Executors, TimeUnit}
import java.util.concurrent.atomic.AtomicLong
diff --git
a/gluten-substrait/src/main/scala/org/apache/gluten/GlutenPlugin.scala
b/gluten-substrait/src/main/scala/org/apache/gluten/GlutenPlugin.scala
index d3179f6bb..66bab72bf 100644
--- a/gluten-substrait/src/main/scala/org/apache/gluten/GlutenPlugin.scala
+++ b/gluten-substrait/src/main/scala/org/apache/gluten/GlutenPlugin.scala
@@ -21,8 +21,8 @@ import org.apache.gluten.backendsapi.BackendsApiManager
import org.apache.gluten.events.GlutenBuildInfoEvent
import org.apache.gluten.exception.GlutenException
import
org.apache.gluten.extension.GlutenSessionExtensions.{GLUTEN_SESSION_EXTENSION_NAME,
SPARK_SESSION_EXTS_KEY}
+import org.apache.gluten.task.TaskListener
import org.apache.gluten.test.TestStats
-import org.apache.gluten.utils.TaskListener
import org.apache.spark.{HdfsConfGenerator, SparkConf, SparkContext,
TaskFailedReason}
import org.apache.spark.api.plugin.{DriverPlugin, ExecutorPlugin,
PluginContext, SparkPlugin}
@@ -31,7 +31,8 @@ import org.apache.spark.listener.GlutenListenerFactory
import org.apache.spark.network.util.JavaUtils
import org.apache.spark.sql.execution.ui.GlutenEventUtils
import org.apache.spark.sql.internal.SQLConf
-import org.apache.spark.util.{SparkResourceUtil, TaskResources}
+import org.apache.spark.task.TaskResources
+import org.apache.spark.util.SparkResourceUtil
import java.util
import java.util.Collections
diff --git
a/gluten-substrait/src/main/scala/org/apache/gluten/planner/plan/GlutenPlanModel.scala
b/gluten-substrait/src/main/scala/org/apache/gluten/planner/plan/GlutenPlanModel.scala
index d981de804..5df4f572c 100644
---
a/gluten-substrait/src/main/scala/org/apache/gluten/planner/plan/GlutenPlanModel.scala
+++
b/gluten-substrait/src/main/scala/org/apache/gluten/planner/plan/GlutenPlanModel.scala
@@ -28,7 +28,8 @@ import org.apache.spark.rdd.RDD
import org.apache.spark.sql.catalyst.InternalRow
import org.apache.spark.sql.catalyst.expressions.Attribute
import org.apache.spark.sql.execution.{ColumnarToRowExec, LeafExecNode,
SparkPlan}
-import org.apache.spark.util.{SparkTaskUtil, TaskResources}
+import org.apache.spark.task.TaskResources
+import org.apache.spark.util.SparkTaskUtil
import java.util.{Objects, Properties}
diff --git
a/gluten-substrait/src/main/scala/org/apache/gluten/utils/iterator/IteratorsV1.scala
b/gluten-substrait/src/main/scala/org/apache/gluten/utils/iterator/IteratorsV1.scala
index 3e9248c44..1090c6944 100644
---
a/gluten-substrait/src/main/scala/org/apache/gluten/utils/iterator/IteratorsV1.scala
+++
b/gluten-substrait/src/main/scala/org/apache/gluten/utils/iterator/IteratorsV1.scala
@@ -19,7 +19,7 @@ package org.apache.gluten.utils.iterator
import org.apache.gluten.utils.iterator.Iterators.WrapperBuilder
import org.apache.spark.{InterruptibleIterator, TaskContext}
-import org.apache.spark.util.TaskResources
+import org.apache.spark.task.TaskResources
import java.util.concurrent.TimeUnit
import java.util.concurrent.atomic.AtomicBoolean
diff --git
a/gluten-substrait/src/test/java/org/apache/gluten/memory/memtarget/spark/TreeMemoryConsumerTest.java
b/gluten-substrait/src/test/java/org/apache/gluten/memory/memtarget/spark/TreeMemoryConsumerTest.java
index bbc43ba5d..befe44918 100644
---
a/gluten-substrait/src/test/java/org/apache/gluten/memory/memtarget/spark/TreeMemoryConsumerTest.java
+++
b/gluten-substrait/src/test/java/org/apache/gluten/memory/memtarget/spark/TreeMemoryConsumerTest.java
@@ -24,7 +24,7 @@ import org.apache.gluten.memory.memtarget.TreeMemoryTarget;
import org.apache.spark.TaskContext;
import org.apache.spark.sql.internal.SQLConf;
-import org.apache.spark.util.TaskResources$;
+import org.apache.spark.task.TaskResources$;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
diff --git
a/gluten-substrait/src/test/scala/org/apache/gluten/utils/TaskResourceSuite.scala
b/gluten-substrait/src/test/scala/org/apache/gluten/utils/TaskResourceSuite.scala
index 0c91b4faf..47ffc8812 100644
---
a/gluten-substrait/src/test/scala/org/apache/gluten/utils/TaskResourceSuite.scala
+++
b/gluten-substrait/src/test/scala/org/apache/gluten/utils/TaskResourceSuite.scala
@@ -19,7 +19,8 @@ package org.apache.gluten.utils
import org.apache.spark.memory.{MemoryConsumer, MemoryMode}
import org.apache.spark.sql.catalyst.plans.SQLHelper
import org.apache.spark.sql.internal.SQLConf
-import org.apache.spark.util.{SparkTaskUtil, TaskResource, TaskResources}
+import org.apache.spark.task.{TaskResource, TaskResources}
+import org.apache.spark.util.SparkTaskUtil
import org.scalatest.funsuite.AnyFunSuite
diff --git
a/gluten-substrait/src/test/scala/org/apache/gluten/utils/iterator/IteratorSuite.scala
b/gluten-substrait/src/test/scala/org/apache/gluten/utils/iterator/IteratorSuite.scala
index 1a84d6719..9a70f235a 100644
---
a/gluten-substrait/src/test/scala/org/apache/gluten/utils/iterator/IteratorSuite.scala
+++
b/gluten-substrait/src/test/scala/org/apache/gluten/utils/iterator/IteratorSuite.scala
@@ -18,7 +18,7 @@ package org.apache.gluten.utils.iterator
import org.apache.gluten.utils.iterator.Iterators.{V1, WrapperBuilder}
-import org.apache.spark.util.TaskResources
+import org.apache.spark.task.TaskResources
import org.scalatest.funsuite.AnyFunSuite
diff --git
a/gluten-substrait/src/test/scala/org/apache/spark/utils/iterator/IteratorBenchmark.scala
b/gluten-substrait/src/test/scala/org/apache/spark/utils/iterator/IteratorBenchmark.scala
index aa69f309a..31f4848f3 100644
---
a/gluten-substrait/src/test/scala/org/apache/spark/utils/iterator/IteratorBenchmark.scala
+++
b/gluten-substrait/src/test/scala/org/apache/spark/utils/iterator/IteratorBenchmark.scala
@@ -20,7 +20,7 @@ import org.apache.gluten.utils.iterator.Iterators
import org.apache.gluten.utils.iterator.Iterators.V1
import org.apache.spark.benchmark.{Benchmark, BenchmarkBase}
-import org.apache.spark.util.TaskResources
+import org.apache.spark.task.TaskResources
object IteratorBenchmark extends BenchmarkBase {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
