This is an automated email from the ASF dual-hosted git repository.
maxgekk pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new e822775f1f07 [SPARK-47243][SS] Correct the package name of
`StateMetadataSource.scala`
e822775f1f07 is described below
commit e822775f1f07dcf13c8f8be5ca54ed2e9681b2a5
Author: yangjie01 <[email protected]>
AuthorDate: Fri Mar 1 22:06:47 2024 +0300
[SPARK-47243][SS] Correct the package name of `StateMetadataSource.scala`
### What changes were proposed in this pull request?
The package name for `StateMetadataSource.scala` is defined as
`org.apache.spark.sql.execution.datasources.v2.state`, but it is in the
`org/apache/spark/sql/execution/datasources/v2/state/metadata` directory. This
pr corrects its package name to
`org.apache.spark.sql.execution.datasources.v2.state.metadata` to ensure
consistency.
### Why are the changes needed?
Correct package name of `StateMetadataSource.scala`
### Does this PR introduce _any_ user-facing change?
Yes, the package name of `StateMetadataSource.scala` changed from
`org.apache.spark.sql.execution.datasources.v2.state` to
`org.apache.spark.sql.execution.datasources.v2.state.metadata` , but this
feature was added in Apache Spark 4.0, it has not been released yet.
### How was this patch tested?
Pass Github Actions
### Was this patch authored or co-authored using generative AI tooling?
No
Closes #45352 from LuciferYang/correct-package-name-StateMetadataSource.
Authored-by: yangjie01 <[email protected]>
Signed-off-by: Max Gekk <[email protected]>
---
.../META-INF/services/org.apache.spark.sql.sources.DataSourceRegister | 2 +-
.../sql/execution/datasources/v2/state/StatePartitionReader.scala | 1 +
.../execution/datasources/v2/state/metadata/StateMetadataSource.scala | 3 ++-
.../apache/spark/sql/execution/streaming/IncrementalExecution.scala | 2 +-
4 files changed, 5 insertions(+), 3 deletions(-)
diff --git
a/sql/core/src/main/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister
b/sql/core/src/main/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister
index 9e32329beb9d..b628c753a767 100644
---
a/sql/core/src/main/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister
+++
b/sql/core/src/main/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister
@@ -28,5 +28,5 @@
org.apache.spark.sql.execution.streaming.sources.RateStreamProvider
org.apache.spark.sql.execution.streaming.sources.TextSocketSourceProvider
org.apache.spark.sql.execution.datasources.binaryfile.BinaryFileFormat
org.apache.spark.sql.execution.streaming.sources.RatePerMicroBatchProvider
-org.apache.spark.sql.execution.datasources.v2.state.StateMetadataSource
+org.apache.spark.sql.execution.datasources.v2.state.metadata.StateMetadataSource
org.apache.spark.sql.execution.datasources.v2.state.StateDataSource
diff --git
a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/state/StatePartitionReader.scala
b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/state/StatePartitionReader.scala
index d9fbb272ecbe..e4e28d5f8ac3 100644
---
a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/state/StatePartitionReader.scala
+++
b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/state/StatePartitionReader.scala
@@ -20,6 +20,7 @@ import org.apache.spark.internal.Logging
import org.apache.spark.sql.catalyst.InternalRow
import org.apache.spark.sql.catalyst.expressions.{GenericInternalRow,
UnsafeRow}
import org.apache.spark.sql.connector.read.{InputPartition, PartitionReader,
PartitionReaderFactory}
+import
org.apache.spark.sql.execution.datasources.v2.state.metadata.StateMetadataPartitionReader
import org.apache.spark.sql.execution.datasources.v2.state.utils.SchemaUtil
import org.apache.spark.sql.execution.streaming.state.{ReadStateStore,
StateStoreConf, StateStoreId, StateStoreProvider, StateStoreProviderId}
import org.apache.spark.sql.types.StructType
diff --git
a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/state/metadata/StateMetadataSource.scala
b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/state/metadata/StateMetadataSource.scala
index 476a5e8e15ca..606a0ac6bc12 100644
---
a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/state/metadata/StateMetadataSource.scala
+++
b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/state/metadata/StateMetadataSource.scala
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.spark.sql.execution.datasources.v2.state
+package org.apache.spark.sql.execution.datasources.v2.state.metadata
import java.util
@@ -28,6 +28,7 @@ import org.apache.spark.sql.catalyst.InternalRow
import org.apache.spark.sql.connector.catalog.{MetadataColumn,
SupportsMetadataColumns, SupportsRead, Table, TableCapability, TableProvider}
import org.apache.spark.sql.connector.expressions.Transform
import org.apache.spark.sql.connector.read.{Batch, InputPartition,
PartitionReader, PartitionReaderFactory, Scan, ScanBuilder}
+import
org.apache.spark.sql.execution.datasources.v2.state.StateDataSourceErrors
import
org.apache.spark.sql.execution.datasources.v2.state.StateSourceOptions.PATH
import org.apache.spark.sql.execution.streaming.CheckpointFileManager
import org.apache.spark.sql.execution.streaming.state.{OperatorStateMetadata,
OperatorStateMetadataReader, OperatorStateMetadataV1}
diff --git
a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/IncrementalExecution.scala
b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/IncrementalExecution.scala
index c343d68c54b7..14007eb4b101 100644
---
a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/IncrementalExecution.scala
+++
b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/IncrementalExecution.scala
@@ -32,7 +32,7 @@ import org.apache.spark.sql.catalyst.trees.TreePattern._
import org.apache.spark.sql.errors.QueryExecutionErrors
import org.apache.spark.sql.execution.{LocalLimitExec, QueryExecution,
SparkPlan, SparkPlanner, UnaryExecNode}
import org.apache.spark.sql.execution.aggregate.{HashAggregateExec,
MergingSessionsExec, ObjectHashAggregateExec, SortAggregateExec,
UpdatingSessionsExec}
-import
org.apache.spark.sql.execution.datasources.v2.state.StateMetadataPartitionReader
+import
org.apache.spark.sql.execution.datasources.v2.state.metadata.StateMetadataPartitionReader
import org.apache.spark.sql.execution.exchange.ShuffleExchangeLike
import org.apache.spark.sql.execution.python.FlatMapGroupsInPandasWithStateExec
import
org.apache.spark.sql.execution.streaming.sources.WriteToMicroBatchDataSourceV1
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
