This is an automated email from the ASF dual-hosted git repository.
lzljs3620320 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new a7bab73e1 [core] Update FileFormatType to String to support user
define FileFormat (#3298)
a7bab73e1 is described below
commit a7bab73e18eb9c1842c78976647b5a5f322f2da4
Author: YanZhangN <[email protected]>
AuthorDate: Tue May 7 12:02:39 2024 +0800
[core] Update FileFormatType to String to support user define FileFormat
(#3298)
---
.../shortcodes/generated/core_configuration.html | 12 ++--
.../paimon/benchmark/TableReadBenchmark.java | 6 +-
.../paimon/benchmark/TableWriterBenchmark.java | 10 ++--
.../main/java/org/apache/paimon/CoreOptions.java | 65 +++++-----------------
.../org/apache/paimon/schema/SchemaValidation.java | 5 +-
.../org/apache/paimon/table/source/RawFile.java | 6 +-
.../java/org/apache/paimon/FileFormatTest.java | 25 +++++----
.../test/java/org/apache/paimon/TestFileStore.java | 4 +-
.../apache/paimon/io/RollingFileWriterTest.java | 13 +++--
.../apache/paimon/schema/SchemaManagerTest.java | 20 -------
.../paimon/table/FileStoreTableTestBase.java | 4 +-
.../paimon/table/PrimaryKeyFileStoreTableTest.java | 2 +-
.../table/source/snapshot/SnapshotReaderTest.java | 2 +-
.../org/apache/paimon/flink/FileStoreITCase.java | 2 +-
.../org/apache/paimon/hive/HiveWriteITCase.java | 8 +--
.../org/apache/paimon/hive/HiveReadITCase.java | 2 +-
.../org/apache/paimon/hive/HiveWriteITCase.java | 10 ++--
.../paimon/hive/PaimonStorageHandlerITCase.java | 24 ++++----
.../paimon/hive/mapred/PaimonRecordReaderTest.java | 4 +-
19 files changed, 82 insertions(+), 142 deletions(-)
diff --git a/docs/layouts/shortcodes/generated/core_configuration.html
b/docs/layouts/shortcodes/generated/core_configuration.html
index 892fce56d..77710afab 100644
--- a/docs/layouts/shortcodes/generated/core_configuration.html
+++ b/docs/layouts/shortcodes/generated/core_configuration.html
@@ -244,9 +244,9 @@ under the License.
</tr>
<tr>
<td><h5>file.format</h5></td>
- <td style="word-wrap: break-word;">orc</td>
- <td><p>Enum</p></td>
- <td>Specify the message format of data files, currently orc,
parquet and avro are supported.<br /><br />Possible values:<ul><li>"orc": ORC
file format.</li><li>"parquet": Parquet file format.</li><li>"avro": Avro file
format.</li></ul></td>
+ <td style="word-wrap: break-word;">"orc"</td>
+ <td>String</td>
+ <td>Specify the message format of data files, currently orc,
parquet and avro are supported.</td>
</tr>
<tr>
<td><h5>file.format.per.level</h5></td>
@@ -341,9 +341,9 @@ Mainly to resolve data skew on primary keys. We recommend
starting with 64 mb wh
</tr>
<tr>
<td><h5>manifest.format</h5></td>
- <td style="word-wrap: break-word;">avro</td>
- <td><p>Enum</p></td>
- <td>Specify the message format of manifest files.<br /><br
/>Possible values:<ul><li>"orc": ORC file format.</li><li>"parquet": Parquet
file format.</li><li>"avro": Avro file format.</li></ul></td>
+ <td style="word-wrap: break-word;">"avro"</td>
+ <td>String</td>
+ <td>Specify the message format of manifest files.</td>
</tr>
<tr>
<td><h5>manifest.full-compaction-threshold-size</h5></td>
diff --git
a/paimon-benchmark/paimon-micro-benchmarks/src/test/java/org/apache/paimon/benchmark/TableReadBenchmark.java
b/paimon-benchmark/paimon-micro-benchmarks/src/test/java/org/apache/paimon/benchmark/TableReadBenchmark.java
index f0af590a0..4047b0d57 100644
---
a/paimon-benchmark/paimon-micro-benchmarks/src/test/java/org/apache/paimon/benchmark/TableReadBenchmark.java
+++
b/paimon-benchmark/paimon-micro-benchmarks/src/test/java/org/apache/paimon/benchmark/TableReadBenchmark.java
@@ -80,19 +80,19 @@ public class TableReadBenchmark extends TableBenchmark {
private Options orc() {
Options options = new Options();
- options.set(CoreOptions.FILE_FORMAT, CoreOptions.FileFormatType.ORC);
+ options.set(CoreOptions.FILE_FORMAT, CoreOptions.FILE_FORMAT_ORC);
return options;
}
private Options parquet() {
Options options = new Options();
- options.set(CoreOptions.FILE_FORMAT,
CoreOptions.FileFormatType.PARQUET);
+ options.set(CoreOptions.FILE_FORMAT, CoreOptions.FILE_FORMAT_ORC);
return options;
}
private Options avro() {
Options options = new Options();
- options.set(CoreOptions.FILE_FORMAT, CoreOptions.FileFormatType.AVRO);
+ options.set(CoreOptions.FILE_FORMAT, CoreOptions.FILE_FORMAT_AVRO);
return options;
}
diff --git
a/paimon-benchmark/paimon-micro-benchmarks/src/test/java/org/apache/paimon/benchmark/TableWriterBenchmark.java
b/paimon-benchmark/paimon-micro-benchmarks/src/test/java/org/apache/paimon/benchmark/TableWriterBenchmark.java
index 9812afebb..d3624f933 100644
---
a/paimon-benchmark/paimon-micro-benchmarks/src/test/java/org/apache/paimon/benchmark/TableWriterBenchmark.java
+++
b/paimon-benchmark/paimon-micro-benchmarks/src/test/java/org/apache/paimon/benchmark/TableWriterBenchmark.java
@@ -35,7 +35,7 @@ public class TableWriterBenchmark extends TableBenchmark {
@Test
public void testAvro() throws Exception {
Options options = new Options();
- options.set(CoreOptions.FILE_FORMAT, CoreOptions.FileFormatType.AVRO);
+ options.set(CoreOptions.FILE_FORMAT, CoreOptions.FILE_FORMAT_AVRO);
innerTest("avro", options);
/*
* Java HotSpot(TM) 64-Bit Server VM 1.8.0_301-b09 on Mac OS X 10.16
@@ -49,7 +49,7 @@ public class TableWriterBenchmark extends TableBenchmark {
@Test
public void testAvroWithoutStats() throws Exception {
Options options = new Options();
- options.set(CoreOptions.FILE_FORMAT, CoreOptions.FileFormatType.AVRO);
+ options.set(CoreOptions.FILE_FORMAT, CoreOptions.FILE_FORMAT_AVRO);
options.set(CoreOptions.METADATA_STATS_MODE, "none");
innerTest("avro", options);
/*
@@ -64,7 +64,7 @@ public class TableWriterBenchmark extends TableBenchmark {
@Test
public void testOrcNoCompression() throws Exception {
Options options = new Options();
- options.set(CoreOptions.FILE_FORMAT, CoreOptions.FileFormatType.ORC);
+ options.set(CoreOptions.FILE_FORMAT, CoreOptions.FILE_FORMAT_ORC);
options.set("orc.compress", "none");
innerTest("orc", options);
/*
@@ -79,7 +79,7 @@ public class TableWriterBenchmark extends TableBenchmark {
@Test
public void testParquet() throws Exception {
Options options = new Options();
- options.set(CoreOptions.FILE_FORMAT,
CoreOptions.FileFormatType.PARQUET);
+ options.set(CoreOptions.FILE_FORMAT, CoreOptions.FILE_FORMAT_PARQUET);
innerTest("parquet", options);
/*
* Java HotSpot(TM) 64-Bit Server VM 1.8.0_301-b09 on Mac OS X 10.16
@@ -93,7 +93,7 @@ public class TableWriterBenchmark extends TableBenchmark {
@Test
public void testOrc() throws Exception {
Options options = new Options();
- options.set(CoreOptions.FILE_FORMAT, CoreOptions.FileFormatType.ORC);
+ options.set(CoreOptions.FILE_FORMAT, CoreOptions.FILE_FORMAT_ORC);
innerTest("orc", options);
/*
* Java HotSpot(TM) 64-Bit Server VM 1.8.0_301-b09 on Mac OS X 10.16
diff --git a/paimon-common/src/main/java/org/apache/paimon/CoreOptions.java
b/paimon-common/src/main/java/org/apache/paimon/CoreOptions.java
index 63832638b..008e3e9e2 100644
--- a/paimon-common/src/main/java/org/apache/paimon/CoreOptions.java
+++ b/paimon-common/src/main/java/org/apache/paimon/CoreOptions.java
@@ -112,10 +112,14 @@ public class CoreOptions implements Serializable {
.noDefaultValue()
.withDescription("The file path of this table in the
filesystem.");
- public static final ConfigOption<FileFormatType> FILE_FORMAT =
+ public static final String FILE_FORMAT_ORC = "orc";
+ public static final String FILE_FORMAT_AVRO = "avro";
+ public static final String FILE_FORMAT_PARQUET = "parquet";
+
+ public static final ConfigOption<String> FILE_FORMAT =
key("file.format")
- .enumType(FileFormatType.class)
- .defaultValue(FileFormatType.ORC)
+ .stringType()
+ .defaultValue(FILE_FORMAT_ORC)
.withDescription(
"Specify the message format of data files,
currently orc, parquet and avro are supported.");
@@ -158,10 +162,10 @@ public class CoreOptions implements Serializable {
.defaultValue(true)
.withDescription("Whether enabled read file index.");
- public static final ConfigOption<FileFormatType> MANIFEST_FORMAT =
+ public static final ConfigOption<String> MANIFEST_FORMAT =
key("manifest.format")
- .enumType(FileFormatType.class)
- .defaultValue(FileFormatType.AVRO)
+ .stringType()
+ .defaultValue(CoreOptions.FILE_FORMAT_AVRO)
.withDescription("Specify the message format of manifest
files.");
public static final ConfigOption<MemorySize> MANIFEST_TARGET_FILE_SIZE =
@@ -1182,8 +1186,8 @@ public class CoreOptions implements Serializable {
return new Path(options.get(PATH));
}
- public FileFormatType formatType() {
- return options.get(FILE_FORMAT);
+ public String formatType() {
+ return options.get(FILE_FORMAT).toLowerCase();
}
public FileFormat fileFormat() {
@@ -1218,9 +1222,8 @@ public class CoreOptions implements Serializable {
return options.get(SORT_COMPACTION_SAMPLE_MAGNIFICATION);
}
- public static FileFormat createFileFormat(
- Options options, ConfigOption<FileFormatType> formatOption) {
- String formatIdentifier = options.get(formatOption).toString();
+ public static FileFormat createFileFormat(Options options,
ConfigOption<String> formatOption) {
+ String formatIdentifier = options.get(formatOption);
return FileFormat.getFileFormat(options, formatIdentifier);
}
@@ -1991,46 +1994,6 @@ public class CoreOptions implements Serializable {
}
}
- /** Specifies the file format type for store. */
- public enum FileFormatType implements DescribedEnum {
- ORC("orc", "ORC file format."),
- PARQUET("parquet", "Parquet file format."),
- AVRO("avro", "Avro file format.");
-
- private final String value;
- private final String description;
-
- FileFormatType(String value, String description) {
- this.value = value;
- this.description = description;
- }
-
- @Override
- public String toString() {
- return value;
- }
-
- @Override
- public InlineElement getDescription() {
- return text(description);
- }
-
- @VisibleForTesting
- public static FileFormatType fromValue(String value) {
- for (FileFormatType formatType : FileFormatType.values()) {
- if (formatType.value.equals(value)) {
- return formatType;
- }
- }
- throw new IllegalArgumentException(
- String.format(
- "Invalid format type %s, only support [%s]",
- value,
- StringUtils.join(
-
Arrays.stream(FileFormatType.values()).iterator(), ",")));
- }
- }
-
/** Specifies the type for streaming read. */
public enum StreamingReadMode implements DescribedEnum {
LOG("log", "Reads from the log store."),
diff --git
a/paimon-core/src/main/java/org/apache/paimon/schema/SchemaValidation.java
b/paimon-core/src/main/java/org/apache/paimon/schema/SchemaValidation.java
index fab5f31fd..0e3a85a47 100644
--- a/paimon-core/src/main/java/org/apache/paimon/schema/SchemaValidation.java
+++ b/paimon-core/src/main/java/org/apache/paimon/schema/SchemaValidation.java
@@ -135,11 +135,8 @@ public class SchemaValidation {
+ " should not be larger than "
+ CHANGELOG_NUM_RETAINED_MAX.key());
- // Get the format type here which will try to convert string value to
{@Code
- // FileFormatType}. If the string value is illegal, an exception will
be thrown.
- CoreOptions.FileFormatType fileFormatType = options.formatType();
FileFormat fileFormat =
- FileFormat.fromIdentifier(fileFormatType.name(), new
Options(schema.options()));
+ FileFormat.fromIdentifier(options.formatType(), new
Options(schema.options()));
fileFormat.validateDataFields(new RowType(schema.fields()));
// Check column names in schema
diff --git
a/paimon-core/src/main/java/org/apache/paimon/table/source/RawFile.java
b/paimon-core/src/main/java/org/apache/paimon/table/source/RawFile.java
index 9967586dc..3758d38d1 100644
--- a/paimon-core/src/main/java/org/apache/paimon/table/source/RawFile.java
+++ b/paimon-core/src/main/java/org/apache/paimon/table/source/RawFile.java
@@ -18,7 +18,6 @@
package org.apache.paimon.table.source;
-import org.apache.paimon.CoreOptions;
import org.apache.paimon.annotation.Public;
import org.apache.paimon.io.DataInputView;
import org.apache.paimon.io.DataOutputView;
@@ -67,10 +66,7 @@ public class RawFile {
return length;
}
- /**
- * Format of the file, which is a lower-cased string. See {@link
CoreOptions.FileFormatType} for
- * all possible types.
- */
+ /** Format of the file, which is a lower-cased string. e.g. avro, orc,
parquet. */
public String format() {
return format;
}
diff --git a/paimon-core/src/test/java/org/apache/paimon/FileFormatTest.java
b/paimon-core/src/test/java/org/apache/paimon/FileFormatTest.java
index fc097bee3..5482fb554 100644
--- a/paimon-core/src/test/java/org/apache/paimon/FileFormatTest.java
+++ b/paimon-core/src/test/java/org/apache/paimon/FileFormatTest.java
@@ -36,12 +36,13 @@ import org.apache.paimon.types.RowType;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
-import static org.apache.paimon.format.orc.OrcFileFormatFactory.IDENTIFIER;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
@@ -99,12 +100,13 @@ public class FileFormatTest {
.hasMessage("Unrecognized codec: _unsupported");
}
- @Test
- public void testCreateFileFormat() {
+ @ParameterizedTest
+ @ValueSource(strings = {"orc", "Orc", "ORC"})
+ public void testCreateFileFormat(String identifier) {
Options tableOptions = new Options();
- tableOptions.set(CoreOptions.FILE_FORMAT,
CoreOptions.FileFormatType.fromValue(IDENTIFIER));
+ tableOptions.set(CoreOptions.FILE_FORMAT, identifier);
tableOptions.set(CoreOptions.READ_BATCH_SIZE, 1024);
- tableOptions.setString(IDENTIFIER + ".hello", "world");
+ tableOptions.setString(identifier + ".hello", "world");
FileFormat fileFormat = CoreOptions.createFileFormat(tableOptions,
CoreOptions.FILE_FORMAT);
assertThat(fileFormat instanceof OrcFileFormat).isTrue();
@@ -113,15 +115,16 @@ public class FileFormatTest {
assertThat(orcFileFormat.formatContext().readBatchSize()).isEqualTo(1024);
}
- @Test
- public void testFileFormatOption() {
+ @ParameterizedTest
+ @ValueSource(strings = {"orc", "Orc", "ORC"})
+ public void testFileFormatOption(String identifier) {
Options tableOptions = new Options();
- tableOptions.set(CoreOptions.FILE_FORMAT,
CoreOptions.FileFormatType.fromValue(IDENTIFIER));
+ tableOptions.set(CoreOptions.FILE_FORMAT, identifier);
tableOptions.set(CoreOptions.READ_BATCH_SIZE, 1024);
- tableOptions.setString(IDENTIFIER + ".hello", "world");
+ tableOptions.setString(identifier + ".hello", "world");
FileFormatDiscover fileFormatDiscover =
FileFormatDiscover.of(new CoreOptions(tableOptions));
- FileFormat fileFormat = fileFormatDiscover.discover(IDENTIFIER);
+ FileFormat fileFormat = fileFormatDiscover.discover(identifier);
assertThat(fileFormat instanceof OrcFileFormat).isTrue();
OrcFileFormat orcFileFormat = (OrcFileFormat) fileFormat;
assertThat(orcFileFormat.formatContext().formatOptions().get("hello")).isEqualTo("world");
@@ -130,7 +133,7 @@ public class FileFormatTest {
public FileFormat createFileFormat(String codec) {
Options tableOptions = new Options();
- tableOptions.set(CoreOptions.FILE_FORMAT,
CoreOptions.FileFormatType.AVRO);
+ tableOptions.set(CoreOptions.FILE_FORMAT,
CoreOptions.FILE_FORMAT_AVRO);
tableOptions.setString("avro.codec", codec);
return CoreOptions.createFileFormat(tableOptions,
CoreOptions.FILE_FORMAT);
}
diff --git a/paimon-core/src/test/java/org/apache/paimon/TestFileStore.java
b/paimon-core/src/test/java/org/apache/paimon/TestFileStore.java
index e71f5d281..e2bb15936 100644
--- a/paimon-core/src/test/java/org/apache/paimon/TestFileStore.java
+++ b/paimon-core/src/test/java/org/apache/paimon/TestFileStore.java
@@ -731,8 +731,8 @@ public class TestFileStore extends KeyValueFileStore {
CoreOptions.MANIFEST_TARGET_FILE_SIZE,
MemorySize.parse((ThreadLocalRandom.current().nextInt(16)
+ 1) + "kb"));
- conf.set(CoreOptions.FILE_FORMAT,
CoreOptions.FileFormatType.fromValue(format));
- conf.set(CoreOptions.MANIFEST_FORMAT,
CoreOptions.FileFormatType.fromValue(format));
+ conf.set(CoreOptions.FILE_FORMAT, format);
+ conf.set(CoreOptions.MANIFEST_FORMAT, format);
conf.set(CoreOptions.PATH, root);
conf.set(CoreOptions.BUCKET, numBuckets);
diff --git
a/paimon-core/src/test/java/org/apache/paimon/io/RollingFileWriterTest.java
b/paimon-core/src/test/java/org/apache/paimon/io/RollingFileWriterTest.java
index 1801c2dd0..a213ba597 100644
--- a/paimon-core/src/test/java/org/apache/paimon/io/RollingFileWriterTest.java
+++ b/paimon-core/src/test/java/org/apache/paimon/io/RollingFileWriterTest.java
@@ -36,13 +36,12 @@ import org.apache.paimon.utils.StatsCollectorFactories;
import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
import org.junit.jupiter.api.io.TempDir;
import org.junit.jupiter.params.ParameterizedTest;
-import org.junit.jupiter.params.provider.EnumSource;
+import org.junit.jupiter.params.provider.ValueSource;
import java.io.File;
import java.io.IOException;
import java.util.HashMap;
-import static org.apache.paimon.CoreOptions.FileFormatType;
import static org.assertj.core.api.Assertions.assertThat;
/** Test for {@link RollingFileWriter}. */
@@ -94,11 +93,13 @@ public class RollingFileWriterTest {
}
@ParameterizedTest
- @EnumSource(FileFormatType.class)
- public void testRolling(FileFormatType formatType) throws IOException {
- initialize(formatType.toString());
+ @ValueSource(strings = {"orc", "avro", "parquet"})
+ public void testRolling(String formatType) throws IOException {
+ initialize(formatType);
int checkInterval =
- formatType == FileFormatType.ORC ?
VectorizedRowBatch.DEFAULT_SIZE : 1000;
+ formatType.equals(CoreOptions.FILE_FORMAT_ORC)
+ ? VectorizedRowBatch.DEFAULT_SIZE
+ : 1000;
for (int i = 0; i < 3000; i++) {
rollingFileWriter.write(GenericRow.of(i));
if (i < checkInterval) {
diff --git
a/paimon-core/src/test/java/org/apache/paimon/schema/SchemaManagerTest.java
b/paimon-core/src/test/java/org/apache/paimon/schema/SchemaManagerTest.java
index b5ee8c639..d912c0ce9 100644
--- a/paimon-core/src/test/java/org/apache/paimon/schema/SchemaManagerTest.java
+++ b/paimon-core/src/test/java/org/apache/paimon/schema/SchemaManagerTest.java
@@ -51,7 +51,6 @@ import java.util.concurrent.ThreadLocalRandom;
import java.util.stream.Collectors;
import java.util.stream.IntStream;
-import static org.apache.paimon.CoreOptions.FILE_FORMAT;
import static org.apache.paimon.utils.FailingFileIO.retryArtificialException;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
@@ -298,23 +297,4 @@ public class SchemaManagerTest {
manager.deleteSchema(manager.latest().get().id());
assertThat(manager.latest().get().toString()).isEqualTo(schemaContent);
}
-
- @Test
- public void testInvalidFormatType() {
- Map<String, String> options = new HashMap<>();
- options.put(FILE_FORMAT.key(), "test");
- Schema schema =
- new Schema(
- rowType.getFields(),
- partitionKeys,
- primaryKeys,
- options,
- "append-only table with primary key");
- SchemaManager manager = new SchemaManager(LocalFileIO.create(), path);
- assertThatThrownBy(() -> manager.createTable(schema))
- .isInstanceOf(IllegalArgumentException.class)
- .hasMessage(
- String.format(
- "Could not parse value 'test' for key '%s'.",
FILE_FORMAT.key()));
- }
}
diff --git
a/paimon-core/src/test/java/org/apache/paimon/table/FileStoreTableTestBase.java
b/paimon-core/src/test/java/org/apache/paimon/table/FileStoreTableTestBase.java
index bccf0fe0e..778c871f7 100644
---
a/paimon-core/src/test/java/org/apache/paimon/table/FileStoreTableTestBase.java
+++
b/paimon-core/src/test/java/org/apache/paimon/table/FileStoreTableTestBase.java
@@ -198,7 +198,7 @@ public abstract class FileStoreTableTestBase {
@Test
public void testChangeFormat() throws Exception {
FileStoreTable table =
- createFileStoreTable(conf -> conf.set(FILE_FORMAT,
CoreOptions.FileFormatType.ORC));
+ createFileStoreTable(conf -> conf.set(FILE_FORMAT,
CoreOptions.FILE_FORMAT_ORC));
StreamTableWrite write = table.newWrite(commitUser);
StreamTableCommit commit = table.newCommit(commitUser);
@@ -219,7 +219,7 @@ public abstract class FileStoreTableTestBase {
table =
createFileStoreTable(
- conf -> conf.set(FILE_FORMAT,
CoreOptions.FileFormatType.PARQUET));
+ conf -> conf.set(FILE_FORMAT,
CoreOptions.FILE_FORMAT_PARQUET));
write = table.newWrite(commitUser);
commit = table.newCommit(commitUser);
write.write(rowData(1, 11, 111L));
diff --git
a/paimon-core/src/test/java/org/apache/paimon/table/PrimaryKeyFileStoreTableTest.java
b/paimon-core/src/test/java/org/apache/paimon/table/PrimaryKeyFileStoreTableTest.java
index 43d338eb8..4c33e9893 100644
---
a/paimon-core/src/test/java/org/apache/paimon/table/PrimaryKeyFileStoreTableTest.java
+++
b/paimon-core/src/test/java/org/apache/paimon/table/PrimaryKeyFileStoreTableTest.java
@@ -1336,7 +1336,7 @@ public class PrimaryKeyFileStoreTableTest extends
FileStoreTableTestBase {
FileStoreTable table =
createFileStoreTable(
options -> {
- options.set(FILE_FORMAT,
CoreOptions.FileFormatType.AVRO);
+ options.set(FILE_FORMAT,
CoreOptions.FILE_FORMAT_AVRO);
options.set(SOURCE_SPLIT_OPEN_FILE_COST,
MemorySize.ofBytes(1));
options.set(SOURCE_SPLIT_TARGET_SIZE,
MemorySize.ofKibiBytes(5));
});
diff --git
a/paimon-core/src/test/java/org/apache/paimon/table/source/snapshot/SnapshotReaderTest.java
b/paimon-core/src/test/java/org/apache/paimon/table/source/snapshot/SnapshotReaderTest.java
index 5788e9722..a498b72c9 100644
---
a/paimon-core/src/test/java/org/apache/paimon/table/source/snapshot/SnapshotReaderTest.java
+++
b/paimon-core/src/test/java/org/apache/paimon/table/source/snapshot/SnapshotReaderTest.java
@@ -340,7 +340,7 @@ public class SnapshotReaderTest {
Options options = new Options();
options.set(CoreOptions.BUCKET, 1);
options.set(CoreOptions.NUM_SORTED_RUNS_COMPACTION_TRIGGER, 5);
- options.set(CoreOptions.FILE_FORMAT, CoreOptions.FileFormatType.AVRO);
+ options.set(CoreOptions.FILE_FORMAT, CoreOptions.FILE_FORMAT_AVRO);
Map<String, String> formatPerLevel = new HashMap<>();
formatPerLevel.put("5", "orc");
options.set(CoreOptions.FILE_FORMAT_PER_LEVEL, formatPerLevel);
diff --git
a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/FileStoreITCase.java
b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/FileStoreITCase.java
index dff589e92..2bd265b60 100644
---
a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/FileStoreITCase.java
+++
b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/FileStoreITCase.java
@@ -479,7 +479,7 @@ public class FileStoreITCase extends AbstractTestBase {
FailingFileIO.reset(failingName, 3, 100);
options.set(PATH, FailingFileIO.getFailingPath(failingName,
temporaryPath));
}
- options.set(FILE_FORMAT, CoreOptions.FileFormatType.AVRO);
+ options.set(FILE_FORMAT, CoreOptions.FILE_FORMAT_AVRO);
return options;
}
diff --git
a/paimon-hive/paimon-hive-connector-3.1/src/test/java/org/apache/paimon/hive/HiveWriteITCase.java
b/paimon-hive/paimon-hive-connector-3.1/src/test/java/org/apache/paimon/hive/HiveWriteITCase.java
index 7513621a8..69955a1c2 100644
---
a/paimon-hive/paimon-hive-connector-3.1/src/test/java/org/apache/paimon/hive/HiveWriteITCase.java
+++
b/paimon-hive/paimon-hive-connector-3.1/src/test/java/org/apache/paimon/hive/HiveWriteITCase.java
@@ -98,7 +98,7 @@ public class HiveWriteITCase {
List<String> partitionKeys,
List<InternalRow> data,
String tableName,
- @Nullable CoreOptions.FileFormatType fileFormatType)
+ @Nullable String fileFormatType)
throws Exception {
String path = folder.newFolder().toURI().toString();
String tableNameNotNull =
@@ -109,7 +109,7 @@ public class HiveWriteITCase {
conf.set(CoreOptions.BUCKET, 2);
conf.set(
CoreOptions.FILE_FORMAT,
- fileFormatType == null ? CoreOptions.FileFormatType.AVRO :
fileFormatType);
+ fileFormatType == null ? CoreOptions.FILE_FORMAT_AVRO :
fileFormatType.toString());
Identifier identifier = Identifier.create(DATABASE_NAME,
tableNameNotNull);
Table table =
FileStoreTestUtils.createFileStoreTable(
@@ -211,7 +211,7 @@ public class HiveWriteITCase {
Collections.singletonList("pt"),
emptyData,
"hive_test_table_output",
- CoreOptions.FileFormatType.ORC);
+ CoreOptions.FILE_FORMAT_ORC);
hiveShell.execute(
String.format(
"INSERT INTO %s VALUES (1, '2023-01-13 20:00:01%s',
'2023-12-23')",
@@ -243,7 +243,7 @@ public class HiveWriteITCase {
Collections.singletonList("pt"),
emptyData,
"hive_test_table_output",
- CoreOptions.FileFormatType.ORC);
+ CoreOptions.FILE_FORMAT_ORC);
assertThat(hiveShell.executeQuery("SHOW CREATE TABLE " +
outputTableName))
.contains(" `ltz` timestamp with local time zone COMMENT
'from deserializer')");
diff --git
a/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveReadITCase.java
b/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveReadITCase.java
index 87cbdf485..3f34c3417 100644
---
a/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveReadITCase.java
+++
b/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveReadITCase.java
@@ -121,7 +121,7 @@ public class HiveReadITCase extends HiveTestBase {
Options conf = new Options();
conf.set(CatalogOptions.WAREHOUSE, path);
conf.set(CoreOptions.BUCKET, 2);
- conf.set(CoreOptions.FILE_FORMAT, CoreOptions.FileFormatType.AVRO);
+ conf.set(CoreOptions.FILE_FORMAT, CoreOptions.FILE_FORMAT_AVRO);
RowType.Builder rowType = RowType.builder();
rowType.field("col1", DataTypes.INT());
rowType.field("Col2", DataTypes.STRING());
diff --git
a/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveWriteITCase.java
b/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveWriteITCase.java
index 24f0bd6e2..7293353ab 100644
---
a/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveWriteITCase.java
+++
b/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveWriteITCase.java
@@ -109,7 +109,7 @@ public class HiveWriteITCase extends HiveTestBase {
Options conf = new Options();
conf.set(CatalogOptions.WAREHOUSE, path);
conf.set(CoreOptions.BUCKET, 2);
- conf.set(CoreOptions.FILE_FORMAT, CoreOptions.FileFormatType.AVRO);
+ conf.set(CoreOptions.FILE_FORMAT, CoreOptions.FILE_FORMAT_AVRO);
Identifier identifier = Identifier.create(DATABASE_NAME,
tableNameNotNull);
Table table =
FileStoreTestUtils.createFileStoreTable(
@@ -133,7 +133,7 @@ public class HiveWriteITCase extends HiveTestBase {
Options conf = new Options();
conf.set(CatalogOptions.WAREHOUSE, path);
conf.set(CoreOptions.BUCKET, 2);
- conf.set(CoreOptions.FILE_FORMAT, CoreOptions.FileFormatType.AVRO);
+ conf.set(CoreOptions.FILE_FORMAT, CoreOptions.FILE_FORMAT_AVRO);
Identifier identifier = Identifier.create(DATABASE_NAME,
tableNameNotNull);
Table table =
FileStoreTestUtils.createFileStoreTable(
@@ -203,7 +203,7 @@ public class HiveWriteITCase extends HiveTestBase {
Options conf = new Options();
conf.set(CatalogOptions.WAREHOUSE, path);
conf.set(CoreOptions.BUCKET, 1);
- conf.set(CoreOptions.FILE_FORMAT, CoreOptions.FileFormatType.AVRO);
+ conf.set(CoreOptions.FILE_FORMAT, CoreOptions.FILE_FORMAT_AVRO);
Identifier identifier = Identifier.create(DATABASE_NAME, innerName);
Table table =
FileStoreTestUtils.createFileStoreTable(
@@ -247,7 +247,7 @@ public class HiveWriteITCase extends HiveTestBase {
Options conf = new Options();
conf.set(CatalogOptions.WAREHOUSE, path);
conf.set(CoreOptions.BUCKET, 1);
- conf.set(CoreOptions.FILE_FORMAT, CoreOptions.FileFormatType.AVRO);
+ conf.set(CoreOptions.FILE_FORMAT, CoreOptions.FILE_FORMAT_AVRO);
conf.set(CoreOptions.COMPACTION_MAX_FILE_NUM, maxCompact);
Identifier identifier = Identifier.create(DATABASE_NAME, innerName);
Table table =
@@ -590,7 +590,7 @@ public class HiveWriteITCase extends HiveTestBase {
String tablePath = String.format("%s/test_db.db/hive_test_table",
root);
Options conf = new Options();
conf.set(CatalogOptions.WAREHOUSE, root);
- conf.set(CoreOptions.FILE_FORMAT, CoreOptions.FileFormatType.AVRO);
+ conf.set(CoreOptions.FILE_FORMAT, CoreOptions.FILE_FORMAT_AVRO);
conf.set("bucket", "1");
Table table =
FileStoreTestUtils.createFileStoreTable(
diff --git
a/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/PaimonStorageHandlerITCase.java
b/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/PaimonStorageHandlerITCase.java
index 815e26848..d4340ca48 100644
---
a/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/PaimonStorageHandlerITCase.java
+++
b/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/PaimonStorageHandlerITCase.java
@@ -143,7 +143,7 @@ public class PaimonStorageHandlerITCase {
GenericRow.of(3, 50L,
BinaryString.fromString("Store"), 200L));
Options conf = getBasicConf();
- conf.set(CoreOptions.FILE_FORMAT, CoreOptions.FileFormatType.AVRO);
+ conf.set(CoreOptions.FILE_FORMAT, CoreOptions.FILE_FORMAT_AVRO);
RowType rowType =
RowType.of(
new DataType[] {
@@ -268,7 +268,7 @@ public class PaimonStorageHandlerITCase {
GenericRow.of(1, 30, 200L,
BinaryString.fromString("Store")));
Options conf = getBasicConf();
- conf.set(CoreOptions.FILE_FORMAT, CoreOptions.FileFormatType.AVRO);
+ conf.set(CoreOptions.FILE_FORMAT, CoreOptions.FILE_FORMAT_AVRO);
RowType rowType =
RowType.of(
new DataType[] {
@@ -366,7 +366,7 @@ public class PaimonStorageHandlerITCase {
GenericRow.of(3, 50L,
BinaryString.fromString("Store"), 200L));
Options conf = getBasicConf();
- conf.set(CoreOptions.FILE_FORMAT, CoreOptions.FileFormatType.AVRO);
+ conf.set(CoreOptions.FILE_FORMAT, CoreOptions.FILE_FORMAT_AVRO);
RowType rowType =
RowType.of(
new DataType[] {
@@ -445,7 +445,7 @@ public class PaimonStorageHandlerITCase {
GenericRow.of(1, 30, 500L,
BinaryString.fromString("Store")));
Options conf = getBasicConf();
- conf.set(CoreOptions.FILE_FORMAT, CoreOptions.FileFormatType.AVRO);
+ conf.set(CoreOptions.FILE_FORMAT, CoreOptions.FILE_FORMAT_AVRO);
RowType rowType =
RowType.of(
new DataType[] {
@@ -550,7 +550,7 @@ public class PaimonStorageHandlerITCase {
@Test
public void testReadAllSupportedTypes() throws Exception {
Options conf = getBasicConf();
- conf.set(CoreOptions.FILE_FORMAT, CoreOptions.FileFormatType.AVRO);
+ conf.set(CoreOptions.FILE_FORMAT, CoreOptions.FILE_FORMAT_AVRO);
Table table =
FileStoreTestUtils.createFileStoreTable(
conf,
@@ -662,7 +662,7 @@ public class PaimonStorageHandlerITCase {
public void testPredicatePushDown() throws Exception {
Options conf = getBasicConf();
conf.set(CoreOptions.BUCKET, 1);
- conf.set(CoreOptions.FILE_FORMAT, CoreOptions.FileFormatType.AVRO);
+ conf.set(CoreOptions.FILE_FORMAT, CoreOptions.FILE_FORMAT_AVRO);
Table table =
FileStoreTestUtils.createFileStoreTable(
conf,
@@ -741,10 +741,10 @@ public class PaimonStorageHandlerITCase {
ThreadLocalRandom random = ThreadLocalRandom.current();
Options conf = getBasicConf();
- CoreOptions.FileFormatType fileFormatType =
+ String fileFormatType =
random.nextBoolean()
- ? CoreOptions.FileFormatType.ORC
- : CoreOptions.FileFormatType.PARQUET;
+ ? CoreOptions.FILE_FORMAT_ORC
+ : CoreOptions.FILE_FORMAT_PARQUET;
conf.set(CoreOptions.FILE_FORMAT, fileFormatType);
int precision = random.nextInt(10);
@@ -804,7 +804,7 @@ public class PaimonStorageHandlerITCase {
// the original precision is maintained, but the file format will
affect the result
// parquet stores timestamp with three forms
String fraction;
- if (fileFormatType == CoreOptions.FileFormatType.ORC) {
+ if (fileFormatType.equals(CoreOptions.FILE_FORMAT_ORC)) {
fraction = ".123456789";
} else {
if (precision <= 3) {
@@ -882,8 +882,8 @@ public class PaimonStorageHandlerITCase {
conf.set(
CoreOptions.FILE_FORMAT,
ThreadLocalRandom.current().nextBoolean()
- ? CoreOptions.FileFormatType.ORC
- : CoreOptions.FileFormatType.PARQUET);
+ ? CoreOptions.FILE_FORMAT_ORC
+ : CoreOptions.FILE_FORMAT_PARQUET);
Table table =
FileStoreTestUtils.createFileStoreTable(
conf,
diff --git
a/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/mapred/PaimonRecordReaderTest.java
b/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/mapred/PaimonRecordReaderTest.java
index 97cdc89eb..da34fff07 100644
---
a/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/mapred/PaimonRecordReaderTest.java
+++
b/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/mapred/PaimonRecordReaderTest.java
@@ -61,7 +61,7 @@ public class PaimonRecordReaderTest {
public void testPk() throws Exception {
Options conf = new Options();
conf.set(CatalogOptions.WAREHOUSE, tempDir.toString());
- conf.set(CoreOptions.FILE_FORMAT, CoreOptions.FileFormatType.AVRO);
+ conf.set(CoreOptions.FILE_FORMAT, CoreOptions.FILE_FORMAT_AVRO);
conf.set("bucket", "1");
Table table =
FileStoreTestUtils.createFileStoreTable(
@@ -104,7 +104,7 @@ public class PaimonRecordReaderTest {
public void testProjectionPushdown() throws Exception {
Options conf = new Options();
conf.set(CatalogOptions.WAREHOUSE, tempDir.toString());
- conf.set(CoreOptions.FILE_FORMAT, CoreOptions.FileFormatType.AVRO);
+ conf.set(CoreOptions.FILE_FORMAT, CoreOptions.FILE_FORMAT_AVRO);
Table table =
FileStoreTestUtils.createFileStoreTable(
conf,
