This is an automated email from the ASF dual-hosted git repository.
huaxingao pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg.git
The following commit(s) were added to refs/heads/main by this push:
new a5a0b8af84 Spark: clean up obsoleted/unused writer classes (#13193)
a5a0b8af84 is described below
commit a5a0b8af84362edd876ab88d377c6a608d634098
Author: Steven Zhen Wu <stevenz...@gmail.com>
AuthorDate: Sat May 31 09:56:29 2025 -0700
Spark: clean up obsoleted/unused writer classes (#13193)
* Spark: clean up obsoleted/unused writer classes
* also clean up 3.4 and 3.5
---
.../spark/source/SparkPartitionedFanoutWriter.java | 55 ----------------------
.../spark/source/SparkPartitionedWriter.java | 55 ----------------------
.../spark/source/SparkPartitionedFanoutWriter.java | 55 ----------------------
.../spark/source/SparkPartitionedWriter.java | 55 ----------------------
.../spark/source/SparkPartitionedFanoutWriter.java | 55 ----------------------
.../spark/source/SparkPartitionedWriter.java | 55 ----------------------
6 files changed, 330 deletions(-)
diff --git
a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/SparkPartitionedFanoutWriter.java
b/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/SparkPartitionedFanoutWriter.java
deleted file mode 100644
index c34ad2f3ad..0000000000
---
a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/SparkPartitionedFanoutWriter.java
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.iceberg.spark.source;
-
-import org.apache.iceberg.FileFormat;
-import org.apache.iceberg.PartitionKey;
-import org.apache.iceberg.PartitionSpec;
-import org.apache.iceberg.Schema;
-import org.apache.iceberg.io.FileAppenderFactory;
-import org.apache.iceberg.io.FileIO;
-import org.apache.iceberg.io.OutputFileFactory;
-import org.apache.iceberg.io.PartitionedFanoutWriter;
-import org.apache.spark.sql.catalyst.InternalRow;
-import org.apache.spark.sql.types.StructType;
-
-public class SparkPartitionedFanoutWriter extends
PartitionedFanoutWriter<InternalRow> {
- private final PartitionKey partitionKey;
- private final InternalRowWrapper internalRowWrapper;
-
- public SparkPartitionedFanoutWriter(
- PartitionSpec spec,
- FileFormat format,
- FileAppenderFactory<InternalRow> appenderFactory,
- OutputFileFactory fileFactory,
- FileIO io,
- long targetFileSize,
- Schema schema,
- StructType sparkSchema) {
- super(spec, format, appenderFactory, fileFactory, io, targetFileSize);
- this.partitionKey = new PartitionKey(spec, schema);
- this.internalRowWrapper = new InternalRowWrapper(sparkSchema,
schema.asStruct());
- }
-
- @Override
- protected PartitionKey partition(InternalRow row) {
- partitionKey.partition(internalRowWrapper.wrap(row));
- return partitionKey;
- }
-}
diff --git
a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/SparkPartitionedWriter.java
b/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/SparkPartitionedWriter.java
deleted file mode 100644
index 6904446829..0000000000
---
a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/SparkPartitionedWriter.java
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.iceberg.spark.source;
-
-import org.apache.iceberg.FileFormat;
-import org.apache.iceberg.PartitionKey;
-import org.apache.iceberg.PartitionSpec;
-import org.apache.iceberg.Schema;
-import org.apache.iceberg.io.FileAppenderFactory;
-import org.apache.iceberg.io.FileIO;
-import org.apache.iceberg.io.OutputFileFactory;
-import org.apache.iceberg.io.PartitionedWriter;
-import org.apache.spark.sql.catalyst.InternalRow;
-import org.apache.spark.sql.types.StructType;
-
-public class SparkPartitionedWriter extends PartitionedWriter<InternalRow> {
- private final PartitionKey partitionKey;
- private final InternalRowWrapper internalRowWrapper;
-
- public SparkPartitionedWriter(
- PartitionSpec spec,
- FileFormat format,
- FileAppenderFactory<InternalRow> appenderFactory,
- OutputFileFactory fileFactory,
- FileIO io,
- long targetFileSize,
- Schema schema,
- StructType sparkSchema) {
- super(spec, format, appenderFactory, fileFactory, io, targetFileSize);
- this.partitionKey = new PartitionKey(spec, schema);
- this.internalRowWrapper = new InternalRowWrapper(sparkSchema,
schema.asStruct());
- }
-
- @Override
- protected PartitionKey partition(InternalRow row) {
- partitionKey.partition(internalRowWrapper.wrap(row));
- return partitionKey;
- }
-}
diff --git
a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/source/SparkPartitionedFanoutWriter.java
b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/source/SparkPartitionedFanoutWriter.java
deleted file mode 100644
index c34ad2f3ad..0000000000
---
a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/source/SparkPartitionedFanoutWriter.java
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.iceberg.spark.source;
-
-import org.apache.iceberg.FileFormat;
-import org.apache.iceberg.PartitionKey;
-import org.apache.iceberg.PartitionSpec;
-import org.apache.iceberg.Schema;
-import org.apache.iceberg.io.FileAppenderFactory;
-import org.apache.iceberg.io.FileIO;
-import org.apache.iceberg.io.OutputFileFactory;
-import org.apache.iceberg.io.PartitionedFanoutWriter;
-import org.apache.spark.sql.catalyst.InternalRow;
-import org.apache.spark.sql.types.StructType;
-
-public class SparkPartitionedFanoutWriter extends
PartitionedFanoutWriter<InternalRow> {
- private final PartitionKey partitionKey;
- private final InternalRowWrapper internalRowWrapper;
-
- public SparkPartitionedFanoutWriter(
- PartitionSpec spec,
- FileFormat format,
- FileAppenderFactory<InternalRow> appenderFactory,
- OutputFileFactory fileFactory,
- FileIO io,
- long targetFileSize,
- Schema schema,
- StructType sparkSchema) {
- super(spec, format, appenderFactory, fileFactory, io, targetFileSize);
- this.partitionKey = new PartitionKey(spec, schema);
- this.internalRowWrapper = new InternalRowWrapper(sparkSchema,
schema.asStruct());
- }
-
- @Override
- protected PartitionKey partition(InternalRow row) {
- partitionKey.partition(internalRowWrapper.wrap(row));
- return partitionKey;
- }
-}
diff --git
a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/source/SparkPartitionedWriter.java
b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/source/SparkPartitionedWriter.java
deleted file mode 100644
index 6904446829..0000000000
---
a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/source/SparkPartitionedWriter.java
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.iceberg.spark.source;
-
-import org.apache.iceberg.FileFormat;
-import org.apache.iceberg.PartitionKey;
-import org.apache.iceberg.PartitionSpec;
-import org.apache.iceberg.Schema;
-import org.apache.iceberg.io.FileAppenderFactory;
-import org.apache.iceberg.io.FileIO;
-import org.apache.iceberg.io.OutputFileFactory;
-import org.apache.iceberg.io.PartitionedWriter;
-import org.apache.spark.sql.catalyst.InternalRow;
-import org.apache.spark.sql.types.StructType;
-
-public class SparkPartitionedWriter extends PartitionedWriter<InternalRow> {
- private final PartitionKey partitionKey;
- private final InternalRowWrapper internalRowWrapper;
-
- public SparkPartitionedWriter(
- PartitionSpec spec,
- FileFormat format,
- FileAppenderFactory<InternalRow> appenderFactory,
- OutputFileFactory fileFactory,
- FileIO io,
- long targetFileSize,
- Schema schema,
- StructType sparkSchema) {
- super(spec, format, appenderFactory, fileFactory, io, targetFileSize);
- this.partitionKey = new PartitionKey(spec, schema);
- this.internalRowWrapper = new InternalRowWrapper(sparkSchema,
schema.asStruct());
- }
-
- @Override
- protected PartitionKey partition(InternalRow row) {
- partitionKey.partition(internalRowWrapper.wrap(row));
- return partitionKey;
- }
-}
diff --git
a/spark/v4.0/spark/src/main/java/org/apache/iceberg/spark/source/SparkPartitionedFanoutWriter.java
b/spark/v4.0/spark/src/main/java/org/apache/iceberg/spark/source/SparkPartitionedFanoutWriter.java
deleted file mode 100644
index c34ad2f3ad..0000000000
---
a/spark/v4.0/spark/src/main/java/org/apache/iceberg/spark/source/SparkPartitionedFanoutWriter.java
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.iceberg.spark.source;
-
-import org.apache.iceberg.FileFormat;
-import org.apache.iceberg.PartitionKey;
-import org.apache.iceberg.PartitionSpec;
-import org.apache.iceberg.Schema;
-import org.apache.iceberg.io.FileAppenderFactory;
-import org.apache.iceberg.io.FileIO;
-import org.apache.iceberg.io.OutputFileFactory;
-import org.apache.iceberg.io.PartitionedFanoutWriter;
-import org.apache.spark.sql.catalyst.InternalRow;
-import org.apache.spark.sql.types.StructType;
-
-public class SparkPartitionedFanoutWriter extends
PartitionedFanoutWriter<InternalRow> {
- private final PartitionKey partitionKey;
- private final InternalRowWrapper internalRowWrapper;
-
- public SparkPartitionedFanoutWriter(
- PartitionSpec spec,
- FileFormat format,
- FileAppenderFactory<InternalRow> appenderFactory,
- OutputFileFactory fileFactory,
- FileIO io,
- long targetFileSize,
- Schema schema,
- StructType sparkSchema) {
- super(spec, format, appenderFactory, fileFactory, io, targetFileSize);
- this.partitionKey = new PartitionKey(spec, schema);
- this.internalRowWrapper = new InternalRowWrapper(sparkSchema,
schema.asStruct());
- }
-
- @Override
- protected PartitionKey partition(InternalRow row) {
- partitionKey.partition(internalRowWrapper.wrap(row));
- return partitionKey;
- }
-}
diff --git
a/spark/v4.0/spark/src/main/java/org/apache/iceberg/spark/source/SparkPartitionedWriter.java
b/spark/v4.0/spark/src/main/java/org/apache/iceberg/spark/source/SparkPartitionedWriter.java
deleted file mode 100644
index 6904446829..0000000000
---
a/spark/v4.0/spark/src/main/java/org/apache/iceberg/spark/source/SparkPartitionedWriter.java
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.iceberg.spark.source;
-
-import org.apache.iceberg.FileFormat;
-import org.apache.iceberg.PartitionKey;
-import org.apache.iceberg.PartitionSpec;
-import org.apache.iceberg.Schema;
-import org.apache.iceberg.io.FileAppenderFactory;
-import org.apache.iceberg.io.FileIO;
-import org.apache.iceberg.io.OutputFileFactory;
-import org.apache.iceberg.io.PartitionedWriter;
-import org.apache.spark.sql.catalyst.InternalRow;
-import org.apache.spark.sql.types.StructType;
-
-public class SparkPartitionedWriter extends PartitionedWriter<InternalRow> {
- private final PartitionKey partitionKey;
- private final InternalRowWrapper internalRowWrapper;
-
- public SparkPartitionedWriter(
- PartitionSpec spec,
- FileFormat format,
- FileAppenderFactory<InternalRow> appenderFactory,
- OutputFileFactory fileFactory,
- FileIO io,
- long targetFileSize,
- Schema schema,
- StructType sparkSchema) {
- super(spec, format, appenderFactory, fileFactory, io, targetFileSize);
- this.partitionKey = new PartitionKey(spec, schema);
- this.internalRowWrapper = new InternalRowWrapper(sparkSchema,
schema.asStruct());
- }
-
- @Override
- protected PartitionKey partition(InternalRow row) {
- partitionKey.partition(internalRowWrapper.wrap(row));
- return partitionKey;
- }
-}
