This is an automated email from the ASF dual-hosted git repository.
lzljs3620320 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-paimon.git
The following commit(s) were added to refs/heads/master by this push:
new 0f0ebd93e [core] add read batch size option (#646)
0f0ebd93e is described below
commit 0f0ebd93e467364d8692355008de60b989f4aace
Author: JunZhang <[email protected]>
AuthorDate: Tue Mar 21 13:25:02 2023 +0800
[core] add read batch size option (#646)
* add read batch size option
* add test case
---
.../shortcodes/generated/core_configuration.html | 6 ++++
.../java/org/apache/paimon/format/FileFormat.java | 21 +++++++------
.../apache/paimon/format/FileFormatFactory.java | 21 ++++++++++++-
.../paimon/format/TestFileFormatFactory.java | 34 ++++++++++++++++++++++
.../main/java/org/apache/paimon/CoreOptions.java | 24 +++++++++++++--
.../apache/paimon/format/FileFormatDiscover.java | 5 +++-
.../java/org/apache/paimon/FileFormatTest.java | 2 +-
.../FileStatsExtractingAvroFormatFactory.java | 4 +--
.../paimon/format/avro/AvroFileFormatFactory.java | 5 ++--
.../apache/paimon/format/orc/OrcFileFormat.java | 12 ++++----
.../paimon/format/orc/OrcFileFormatFactory.java | 7 +++--
.../paimon/format/parquet/ParquetFileFormat.java | 16 ++++++----
.../format/parquet/ParquetFileFormatFactory.java | 7 +++--
.../format/parquet/ParquetReaderFactory.java | 6 ----
.../paimon/format/orc/OrcFileFormatTest.java | 9 ++++--
.../format/parquet/ParquetFileFormatTest.java | 13 +++++++--
16 files changed, 144 insertions(+), 48 deletions(-)
diff --git a/docs/layouts/shortcodes/generated/core_configuration.html
b/docs/layouts/shortcodes/generated/core_configuration.html
index ee5860242..9e38a144b 100644
--- a/docs/layouts/shortcodes/generated/core_configuration.html
+++ b/docs/layouts/shortcodes/generated/core_configuration.html
@@ -356,5 +356,11 @@
<td>Boolean</td>
<td>If set to true, compactions and snapshot expiration will be
skipped. This option is used along with dedicated compact jobs.</td>
</tr>
+ <tr>
+ <td><h5>read.batch-size</h5></td>
+ <td style="word-wrap: break-word;">1024</td>
+ <td>Integer</td>
+ <td>Read batch size for orc and parquet.</td>
+ </tr>
</tbody>
</table>
diff --git
a/paimon-common/src/main/java/org/apache/paimon/format/FileFormat.java
b/paimon-common/src/main/java/org/apache/paimon/format/FileFormat.java
index 77862e826..c4ea21ddc 100644
--- a/paimon-common/src/main/java/org/apache/paimon/format/FileFormat.java
+++ b/paimon-common/src/main/java/org/apache/paimon/format/FileFormat.java
@@ -18,7 +18,8 @@
package org.apache.paimon.format;
-import org.apache.paimon.options.ConfigOption;
+import org.apache.paimon.annotation.VisibleForTesting;
+import org.apache.paimon.format.FileFormatFactory.FormatContext;
import org.apache.paimon.options.Options;
import org.apache.paimon.predicate.Predicate;
import org.apache.paimon.types.RowType;
@@ -75,20 +76,18 @@ public abstract class FileFormat {
return Optional.empty();
}
- /** Create a {@link FileFormat} from table options. */
- public static FileFormat fromTableOptions(
- Options tableOptions, ConfigOption<String> formatOption) {
- String formatIdentifier = tableOptions.get(formatOption);
- return fromIdentifier(formatIdentifier,
tableOptions.removePrefix(formatIdentifier + "."));
+ @VisibleForTesting
+ public static FileFormat fromIdentifier(String identifier, Options
options) {
+ return fromIdentifier(identifier, new FormatContext(options, 1024));
}
/** Create a {@link FileFormat} from format identifier and format options.
*/
- public static FileFormat fromIdentifier(String identifier, Options
options) {
+ public static FileFormat fromIdentifier(String identifier, FormatContext
context) {
Optional<FileFormat> format =
- fromIdentifier(identifier, options,
Thread.currentThread().getContextClassLoader());
+ fromIdentifier(identifier, context,
Thread.currentThread().getContextClassLoader());
return format.orElseGet(
() ->
- fromIdentifier(identifier, options,
FileFormat.class.getClassLoader())
+ fromIdentifier(identifier, context,
FileFormat.class.getClassLoader())
.orElseThrow(
() ->
new RuntimeException(
@@ -99,12 +98,12 @@ public abstract class FileFormat {
}
private static Optional<FileFormat> fromIdentifier(
- String formatIdentifier, Options formatOptions, ClassLoader
classLoader) {
+ String formatIdentifier, FormatContext context, ClassLoader
classLoader) {
ServiceLoader<FileFormatFactory> serviceLoader =
ServiceLoader.load(FileFormatFactory.class, classLoader);
for (FileFormatFactory factory : serviceLoader) {
if (factory.identifier().equals(formatIdentifier.toLowerCase())) {
- return Optional.of(factory.create(formatOptions));
+ return Optional.of(factory.create(context));
}
}
diff --git
a/paimon-common/src/main/java/org/apache/paimon/format/FileFormatFactory.java
b/paimon-common/src/main/java/org/apache/paimon/format/FileFormatFactory.java
index bab037d41..348ab968e 100644
---
a/paimon-common/src/main/java/org/apache/paimon/format/FileFormatFactory.java
+++
b/paimon-common/src/main/java/org/apache/paimon/format/FileFormatFactory.java
@@ -25,5 +25,24 @@ public interface FileFormatFactory {
String identifier();
- FileFormat create(Options formatOptions);
+ FileFormat create(FormatContext formatContext);
+
+ /** the format context. */
+ class FormatContext {
+ private final Options formatOptions;
+ private final int readBatchSize;
+
+ public FormatContext(Options formatOptions, int readBatchSize) {
+ this.formatOptions = formatOptions;
+ this.readBatchSize = readBatchSize;
+ }
+
+ public Options formatOptions() {
+ return formatOptions;
+ }
+
+ public int readBatchSize() {
+ return readBatchSize;
+ }
+ }
}
diff --git
a/paimon-common/src/test/java/org/apache/paimon/format/TestFileFormatFactory.java
b/paimon-common/src/test/java/org/apache/paimon/format/TestFileFormatFactory.java
new file mode 100644
index 000000000..67ffbe614
--- /dev/null
+++
b/paimon-common/src/test/java/org/apache/paimon/format/TestFileFormatFactory.java
@@ -0,0 +1,34 @@
+package org.apache.paimon.format;
+
+import org.apache.paimon.options.Options;
+
+import org.junit.jupiter.api.Test;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+public class TestFileFormatFactory implements FileFormatFactory {
+
+ @Override
+ public String identifier() {
+ return "test";
+ }
+
+ @Override
+ public FileFormat create(FormatContext context) {
+ assertThat(context.formatOptions().get("hello")).isEqualTo("world");
+ assertThat(context.readBatchSize()).isEqualTo(1024);
+ throw new SuccessException();
+ }
+
+ public static class SuccessException extends RuntimeException {}
+
+ @Test
+ public void testCreateFileFormat() {
+ Options options = new Options();
+ options.setString("hello", "world");
+ assertThrows(
+ SuccessException.class,
+ () -> new TestFileFormatFactory().create(new
FormatContext(options, 1024)));
+ }
+}
diff --git a/paimon-core/src/main/java/org/apache/paimon/CoreOptions.java
b/paimon-core/src/main/java/org/apache/paimon/CoreOptions.java
index 9da783634..acb986244 100644
--- a/paimon-core/src/main/java/org/apache/paimon/CoreOptions.java
+++ b/paimon-core/src/main/java/org/apache/paimon/CoreOptions.java
@@ -21,6 +21,7 @@ package org.apache.paimon;
import org.apache.paimon.annotation.Documentation.ExcludeFromDocumentation;
import org.apache.paimon.annotation.Documentation.Immutable;
import org.apache.paimon.format.FileFormat;
+import org.apache.paimon.format.FileFormatFactory;
import org.apache.paimon.fs.Path;
import org.apache.paimon.options.ConfigOption;
import org.apache.paimon.options.MemorySize;
@@ -523,6 +524,12 @@ public class CoreOptions implements Serializable {
.defaultValue(MemorySize.parse("256 mb"))
.withDescription("Max memory size for lookup cache.");
+ public static final ConfigOption<Integer> READ_BATCH_SIZE =
+ key("read.batch-size")
+ .intType()
+ .defaultValue(1024)
+ .withDescription("Read batch size for orc and parquet.");
+
private final Options options;
public CoreOptions(Map<String, String> options) {
@@ -558,11 +565,11 @@ public class CoreOptions implements Serializable {
}
public FileFormat fileFormat() {
- return FileFormat.fromTableOptions(options, FILE_FORMAT);
+ return createFileFormat(options, FILE_FORMAT);
}
public FileFormat manifestFormat() {
- return FileFormat.fromTableOptions(options, MANIFEST_FORMAT);
+ return createFileFormat(options, MANIFEST_FORMAT);
}
public MemorySize manifestTargetSize() {
@@ -573,6 +580,15 @@ public class CoreOptions implements Serializable {
return options.get(PARTITION_DEFAULT_NAME);
}
+ public static FileFormat createFileFormat(Options options,
ConfigOption<String> formatOption) {
+ String formatIdentifier = options.get(formatOption);
+ int readBatchSize = options.get(READ_BATCH_SIZE);
+ return FileFormat.fromIdentifier(
+ formatIdentifier,
+ new FileFormatFactory.FormatContext(
+ options.removePrefix(formatIdentifier + "."),
readBatchSize));
+ }
+
public Map<Integer, String> fileCompressionPerLevel() {
Map<String, String> levelCompressions =
options.get(FILE_COMPRESSION_PER_LEVEL);
return levelCompressions.entrySet().stream()
@@ -752,6 +768,10 @@ public class CoreOptions implements Serializable {
return options.get(PARTITION_TIMESTAMP_PATTERN);
}
+ public int readBatchSize() {
+ return options.get(READ_BATCH_SIZE);
+ }
+
/** Specifies the merge engine for table with primary key. */
public enum MergeEngine implements DescribedEnum {
DEDUPLICATE("deduplicate", "De-duplicate and keep the last row."),
diff --git
a/paimon-core/src/main/java/org/apache/paimon/format/FileFormatDiscover.java
b/paimon-core/src/main/java/org/apache/paimon/format/FileFormatDiscover.java
index 4d762e89c..34f9f8df4 100644
--- a/paimon-core/src/main/java/org/apache/paimon/format/FileFormatDiscover.java
+++ b/paimon-core/src/main/java/org/apache/paimon/format/FileFormatDiscover.java
@@ -36,7 +36,10 @@ public interface FileFormatDiscover {
}
private FileFormat create(String identifier) {
- return FileFormat.fromIdentifier(identifier,
options.toConfiguration());
+ return FileFormat.fromIdentifier(
+ identifier,
+ new FileFormatFactory.FormatContext(
+ options.toConfiguration(),
options.readBatchSize()));
}
};
}
diff --git a/paimon-core/src/test/java/org/apache/paimon/FileFormatTest.java
b/paimon-core/src/test/java/org/apache/paimon/FileFormatTest.java
index 28f40424f..cda1f8940 100644
--- a/paimon-core/src/test/java/org/apache/paimon/FileFormatTest.java
+++ b/paimon-core/src/test/java/org/apache/paimon/FileFormatTest.java
@@ -89,6 +89,6 @@ public class FileFormatTest {
Options tableOptions = new Options();
tableOptions.set(CoreOptions.FILE_FORMAT, "avro");
tableOptions.setString("avro.codec", codec);
- return FileFormat.fromTableOptions(tableOptions,
CoreOptions.FILE_FORMAT);
+ return CoreOptions.createFileFormat(tableOptions,
CoreOptions.FILE_FORMAT);
}
}
diff --git
a/paimon-core/src/test/java/org/apache/paimon/format/FileStatsExtractingAvroFormatFactory.java
b/paimon-core/src/test/java/org/apache/paimon/format/FileStatsExtractingAvroFormatFactory.java
index db9c3b0aa..2299914c1 100644
---
a/paimon-core/src/test/java/org/apache/paimon/format/FileStatsExtractingAvroFormatFactory.java
+++
b/paimon-core/src/test/java/org/apache/paimon/format/FileStatsExtractingAvroFormatFactory.java
@@ -18,8 +18,6 @@
package org.apache.paimon.format;
-import org.apache.paimon.options.Options;
-
/** Factory to create {@link FileStatsExtractingAvroFormat}. */
public class FileStatsExtractingAvroFormatFactory implements FileFormatFactory
{
@@ -29,7 +27,7 @@ public class FileStatsExtractingAvroFormatFactory implements
FileFormatFactory {
}
@Override
- public FileFormat create(Options formatOptions) {
+ public FileFormat create(FormatContext formatContext) {
return new FileStatsExtractingAvroFormat();
}
}
diff --git
a/paimon-format/src/main/java/org/apache/paimon/format/avro/AvroFileFormatFactory.java
b/paimon-format/src/main/java/org/apache/paimon/format/avro/AvroFileFormatFactory.java
index f93e73a8e..a1aae2d45 100644
---
a/paimon-format/src/main/java/org/apache/paimon/format/avro/AvroFileFormatFactory.java
+++
b/paimon-format/src/main/java/org/apache/paimon/format/avro/AvroFileFormatFactory.java
@@ -20,7 +20,6 @@ package org.apache.paimon.format.avro;
import org.apache.paimon.format.FileFormat;
import org.apache.paimon.format.FileFormatFactory;
-import org.apache.paimon.options.Options;
/** Factory to create {@link AvroFileFormat}. */
public class AvroFileFormatFactory implements FileFormatFactory {
@@ -31,7 +30,7 @@ public class AvroFileFormatFactory implements
FileFormatFactory {
}
@Override
- public FileFormat create(Options formatOptions) {
- return new AvroFileFormat(formatOptions);
+ public FileFormat create(FormatContext formatContext) {
+ return new AvroFileFormat(formatContext.formatOptions());
}
}
diff --git
a/paimon-format/src/main/java/org/apache/paimon/format/orc/OrcFileFormat.java
b/paimon-format/src/main/java/org/apache/paimon/format/orc/OrcFileFormat.java
index 5c99d4058..8ac95f2bb 100644
---
a/paimon-format/src/main/java/org/apache/paimon/format/orc/OrcFileFormat.java
+++
b/paimon-format/src/main/java/org/apache/paimon/format/orc/OrcFileFormat.java
@@ -21,6 +21,7 @@ package org.apache.paimon.format.orc;
import org.apache.paimon.annotation.VisibleForTesting;
import org.apache.paimon.data.InternalRow;
import org.apache.paimon.format.FileFormat;
+import org.apache.paimon.format.FileFormatFactory.FormatContext;
import org.apache.paimon.format.FileStatsExtractor;
import org.apache.paimon.format.FormatReaderFactory;
import org.apache.paimon.format.FormatWriterFactory;
@@ -42,7 +43,6 @@ import org.apache.paimon.types.MultisetType;
import org.apache.paimon.types.RowType;
import org.apache.paimon.utils.Projection;
-import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
import org.apache.orc.TypeDescription;
import javax.annotation.Nullable;
@@ -64,12 +64,15 @@ public class OrcFileFormat extends FileFormat {
private final org.apache.hadoop.conf.Configuration readerConf;
private final org.apache.hadoop.conf.Configuration writerConf;
- public OrcFileFormat(Options formatOptions) {
+ private final FormatContext formatContext;
+
+ public OrcFileFormat(FormatContext formatContext) {
super(IDENTIFIER);
- this.orcProperties = getOrcProperties(formatOptions);
+ this.orcProperties = getOrcProperties(formatContext.formatOptions());
this.readerConf = new org.apache.hadoop.conf.Configuration();
this.orcProperties.forEach((k, v) -> readerConf.set(k.toString(),
v.toString()));
this.writerConf = new org.apache.hadoop.conf.Configuration();
+ this.formatContext = formatContext;
}
@VisibleForTesting
@@ -100,8 +103,7 @@ public class OrcFileFormat extends FileFormat {
(RowType) refineDataType(type),
Projection.of(projection).toTopLevelIndexes(),
orcPredicates,
- // same as the default value to avoid the orc bug
- VectorizedRowBatch.DEFAULT_SIZE);
+ formatContext.readBatchSize());
}
/**
diff --git
a/paimon-format/src/main/java/org/apache/paimon/format/orc/OrcFileFormatFactory.java
b/paimon-format/src/main/java/org/apache/paimon/format/orc/OrcFileFormatFactory.java
index 10498e7e1..1c316463c 100644
---
a/paimon-format/src/main/java/org/apache/paimon/format/orc/OrcFileFormatFactory.java
+++
b/paimon-format/src/main/java/org/apache/paimon/format/orc/OrcFileFormatFactory.java
@@ -34,8 +34,11 @@ public class OrcFileFormatFactory implements
FileFormatFactory {
}
@Override
- public OrcFileFormat create(Options formatOptions) {
- return new OrcFileFormat(supplyDefaultOptions(formatOptions));
+ public OrcFileFormat create(FormatContext formatContext) {
+ return new OrcFileFormat(
+ new FormatContext(
+ supplyDefaultOptions(formatContext.formatOptions()),
+ formatContext.readBatchSize()));
}
private Options supplyDefaultOptions(Options options) {
diff --git
a/paimon-format/src/main/java/org/apache/paimon/format/parquet/ParquetFileFormat.java
b/paimon-format/src/main/java/org/apache/paimon/format/parquet/ParquetFileFormat.java
index 6785c0a73..f63fe940e 100644
---
a/paimon-format/src/main/java/org/apache/paimon/format/parquet/ParquetFileFormat.java
+++
b/paimon-format/src/main/java/org/apache/paimon/format/parquet/ParquetFileFormat.java
@@ -20,6 +20,7 @@ package org.apache.paimon.format.parquet;
import org.apache.paimon.annotation.VisibleForTesting;
import org.apache.paimon.format.FileFormat;
+import org.apache.paimon.format.FileFormatFactory.FormatContext;
import org.apache.paimon.format.FileStatsExtractor;
import org.apache.paimon.format.FormatReaderFactory;
import org.apache.paimon.format.FormatWriterFactory;
@@ -37,29 +38,32 @@ import static
org.apache.paimon.format.parquet.ParquetFileFormatFactory.IDENTIFI
/** Parquet {@link FileFormat}. */
public class ParquetFileFormat extends FileFormat {
- private final Options formatOptions;
+ private final FormatContext formatContext;
- public ParquetFileFormat(Options formatOptions) {
+ public ParquetFileFormat(FormatContext formatContext) {
super(IDENTIFIER);
- this.formatOptions = formatOptions;
+ this.formatContext = formatContext;
}
@VisibleForTesting
Options formatOptions() {
- return formatOptions;
+ return formatContext.formatOptions();
}
@Override
public FormatReaderFactory createReaderFactory(
RowType type, int[][] projection, List<Predicate> filters) {
return new ParquetReaderFactory(
- getParquetConfiguration(formatOptions),
Projection.of(projection).project(type));
+ getParquetConfiguration(formatContext.formatOptions()),
+ Projection.of(projection).project(type),
+ formatContext.readBatchSize());
}
@Override
public FormatWriterFactory createWriterFactory(RowType type) {
return new ParquetWriterFactory(
- new RowDataParquetBuilder(type,
getParquetConfiguration(formatOptions)));
+ new RowDataParquetBuilder(
+ type,
getParquetConfiguration(formatContext.formatOptions())));
}
@Override
diff --git
a/paimon-format/src/main/java/org/apache/paimon/format/parquet/ParquetFileFormatFactory.java
b/paimon-format/src/main/java/org/apache/paimon/format/parquet/ParquetFileFormatFactory.java
index bbdfa2552..4a6ad9ff0 100644
---
a/paimon-format/src/main/java/org/apache/paimon/format/parquet/ParquetFileFormatFactory.java
+++
b/paimon-format/src/main/java/org/apache/paimon/format/parquet/ParquetFileFormatFactory.java
@@ -36,8 +36,11 @@ public class ParquetFileFormatFactory implements
FileFormatFactory {
}
@Override
- public ParquetFileFormat create(Options formatOptions) {
- return new ParquetFileFormat(supplyDefaultOptions(formatOptions));
+ public ParquetFileFormat create(FormatContext formatContext) {
+ return new ParquetFileFormat(
+ new FormatContext(
+ supplyDefaultOptions(formatContext.formatOptions()),
+ formatContext.readBatchSize()));
}
private Options supplyDefaultOptions(Options options) {
diff --git
a/paimon-format/src/main/java/org/apache/paimon/format/parquet/ParquetReaderFactory.java
b/paimon-format/src/main/java/org/apache/paimon/format/parquet/ParquetReaderFactory.java
index 24ed57c33..22c023167 100644
---
a/paimon-format/src/main/java/org/apache/paimon/format/parquet/ParquetReaderFactory.java
+++
b/paimon-format/src/main/java/org/apache/paimon/format/parquet/ParquetReaderFactory.java
@@ -73,18 +73,12 @@ public class ParquetReaderFactory implements
FormatReaderFactory {
private static final String ALLOCATION_SIZE =
"parquet.read.allocation.size";
- private static final int BATCH_SIZE = 2048;
-
private final Options conf;
private final String[] projectedFields;
private final DataType[] projectedTypes;
private final int batchSize;
private final Set<Integer> unknownFieldsIndices = new HashSet<>();
- public ParquetReaderFactory(Options formatConfig, RowType projectedType) {
- this(formatConfig, projectedType, BATCH_SIZE);
- }
-
public ParquetReaderFactory(Options conf, RowType projectedType, int
batchSize) {
this.conf = conf;
this.projectedFields = projectedType.getFieldNames().toArray(new
String[0]);
diff --git
a/paimon-format/src/test/java/org/apache/paimon/format/orc/OrcFileFormatTest.java
b/paimon-format/src/test/java/org/apache/paimon/format/orc/OrcFileFormatTest.java
index 335854c5f..42fe80ce4 100644
---
a/paimon-format/src/test/java/org/apache/paimon/format/orc/OrcFileFormatTest.java
+++
b/paimon-format/src/test/java/org/apache/paimon/format/orc/OrcFileFormatTest.java
@@ -18,6 +18,7 @@
package org.apache.paimon.format.orc;
+import org.apache.paimon.format.FileFormatFactory;
import org.apache.paimon.options.Options;
import org.junit.jupiter.api.Test;
@@ -32,7 +33,9 @@ public class OrcFileFormatTest {
public void testAbsent() {
Options options = new Options();
options.setString("haha", "1");
- OrcFileFormat orc = new OrcFileFormatFactory().create(options);
+ OrcFileFormat orc =
+ new OrcFileFormatFactory()
+ .create(new FileFormatFactory.FormatContext(options,
1024));
assertThat(orc.orcProperties().getProperty(IDENTIFIER + ".haha",
"")).isEqualTo("1");
assertThat(orc.orcProperties().getProperty(IDENTIFIER + ".compress",
"")).isEqualTo("lz4");
}
@@ -42,7 +45,9 @@ public class OrcFileFormatTest {
Options options = new Options();
options.setString("haha", "1");
options.setString("compress", "zlib");
- OrcFileFormat orc = new OrcFileFormatFactory().create(options);
+ OrcFileFormat orc =
+ new OrcFileFormatFactory()
+ .create(new FileFormatFactory.FormatContext(options,
1024));
assertThat(orc.orcProperties().getProperty(IDENTIFIER + ".haha",
"")).isEqualTo("1");
assertThat(orc.orcProperties().getProperty(IDENTIFIER + ".compress",
"")).isEqualTo("zlib");
}
diff --git
a/paimon-format/src/test/java/org/apache/paimon/format/parquet/ParquetFileFormatTest.java
b/paimon-format/src/test/java/org/apache/paimon/format/parquet/ParquetFileFormatTest.java
index 0f238da66..f67a91abb 100644
---
a/paimon-format/src/test/java/org/apache/paimon/format/parquet/ParquetFileFormatTest.java
+++
b/paimon-format/src/test/java/org/apache/paimon/format/parquet/ParquetFileFormatTest.java
@@ -18,6 +18,7 @@
package org.apache.paimon.format.parquet;
+import org.apache.paimon.format.FileFormatFactory;
import org.apache.paimon.options.ConfigOption;
import org.apache.paimon.options.ConfigOptions;
import org.apache.paimon.options.Options;
@@ -38,7 +39,9 @@ public class ParquetFileFormatTest {
@Test
public void testAbsent() {
Options options = new Options();
- ParquetFileFormat parquet = new
ParquetFileFormatFactory().create(options);
+ ParquetFileFormat parquet =
+ new ParquetFileFormatFactory()
+ .create(new FileFormatFactory.FormatContext(options,
1024));
assertThat(parquet.formatOptions().getString(KEY1)).isEqualTo("absent");
}
@@ -46,7 +49,9 @@ public class ParquetFileFormatTest {
public void testPresent() {
Options options = new Options();
options.setString(KEY1.key(), "v1");
- ParquetFileFormat parquet = new
ParquetFileFormatFactory().create(options);
+ ParquetFileFormat parquet =
+ new ParquetFileFormatFactory()
+ .create(new FileFormatFactory.FormatContext(options,
1024));
assertThat(parquet.formatOptions().getString(KEY1)).isEqualTo("v1");
}
@@ -66,7 +71,9 @@ public class ParquetFileFormatTest {
private String getCompressionCodec(Options conf) {
Options formatOptions = conf.removePrefix(IDENTIFIER + ".");
- ParquetFileFormat parquet = new
ParquetFileFormatFactory().create(formatOptions);
+ ParquetFileFormat parquet =
+ new ParquetFileFormatFactory()
+ .create(new
FileFormatFactory.FormatContext(formatOptions, 1024));
return getParquetConfiguration(parquet.formatOptions())
.getString(ParquetOutputFormat.COMPRESSION, null);
}
