pvary commented on code in PR #14353:
URL: https://github.com/apache/iceberg/pull/14353#discussion_r2438845003
##########
spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderWithBloomFilter.java:
##########
@@ -248,91 +248,90 @@ protected void dropTable(String name) {
private DataFile writeDataFile(OutputFile out, StructLike partition,
List<Record> rows)
throws IOException {
FileFormat format = defaultFormat(table.properties());
- GenericAppenderFactory factory = new
GenericAppenderFactory(table.schema(), table.spec());
+ Map<String, String> writeProperties = Maps.newHashMap();
boolean useBloomFilterCol1 =
PropertyUtil.propertyAsBoolean(
table.properties(), PARQUET_BLOOM_FILTER_COLUMN_ENABLED_PREFIX +
"id", false);
- factory.set(
+ writeProperties.put(
PARQUET_BLOOM_FILTER_COLUMN_ENABLED_PREFIX + "id",
Boolean.toString(useBloomFilterCol1));
boolean useBloomFilterCol2 =
PropertyUtil.propertyAsBoolean(
table.properties(), PARQUET_BLOOM_FILTER_COLUMN_ENABLED_PREFIX +
"id_long", false);
- factory.set(
+ writeProperties.put(
PARQUET_BLOOM_FILTER_COLUMN_ENABLED_PREFIX + "id_long",
Boolean.toString(useBloomFilterCol2));
boolean useBloomFilterCol3 =
PropertyUtil.propertyAsBoolean(
table.properties(), PARQUET_BLOOM_FILTER_COLUMN_ENABLED_PREFIX +
"id_double", false);
- factory.set(
+ writeProperties.put(
PARQUET_BLOOM_FILTER_COLUMN_ENABLED_PREFIX + "id_double",
Boolean.toString(useBloomFilterCol3));
boolean useBloomFilterCol4 =
PropertyUtil.propertyAsBoolean(
table.properties(), PARQUET_BLOOM_FILTER_COLUMN_ENABLED_PREFIX +
"id_float", false);
- factory.set(
+ writeProperties.put(
PARQUET_BLOOM_FILTER_COLUMN_ENABLED_PREFIX + "id_float",
Boolean.toString(useBloomFilterCol4));
boolean useBloomFilterCol5 =
PropertyUtil.propertyAsBoolean(
table.properties(), PARQUET_BLOOM_FILTER_COLUMN_ENABLED_PREFIX +
"id_string", false);
- factory.set(
+ writeProperties.put(
PARQUET_BLOOM_FILTER_COLUMN_ENABLED_PREFIX + "id_string",
Boolean.toString(useBloomFilterCol5));
boolean useBloomFilterCol6 =
PropertyUtil.propertyAsBoolean(
table.properties(), PARQUET_BLOOM_FILTER_COLUMN_ENABLED_PREFIX +
"id_boolean", false);
- factory.set(
+ writeProperties.put(
PARQUET_BLOOM_FILTER_COLUMN_ENABLED_PREFIX + "id_boolean",
Boolean.toString(useBloomFilterCol6));
boolean useBloomFilterCol7 =
PropertyUtil.propertyAsBoolean(
table.properties(), PARQUET_BLOOM_FILTER_COLUMN_ENABLED_PREFIX +
"id_date", false);
- factory.set(
+ writeProperties.put(
PARQUET_BLOOM_FILTER_COLUMN_ENABLED_PREFIX + "id_date",
Boolean.toString(useBloomFilterCol7));
boolean useBloomFilterCol8 =
PropertyUtil.propertyAsBoolean(
table.properties(),
PARQUET_BLOOM_FILTER_COLUMN_ENABLED_PREFIX + "id_int_decimal",
false);
- factory.set(
+ writeProperties.put(
PARQUET_BLOOM_FILTER_COLUMN_ENABLED_PREFIX + "id_int_decimal",
Boolean.toString(useBloomFilterCol8));
boolean useBloomFilterCol9 =
PropertyUtil.propertyAsBoolean(
table.properties(),
PARQUET_BLOOM_FILTER_COLUMN_ENABLED_PREFIX + "id_long_decimal",
false);
- factory.set(
+ writeProperties.put(
PARQUET_BLOOM_FILTER_COLUMN_ENABLED_PREFIX + "id_long_decimal",
Boolean.toString(useBloomFilterCol9));
boolean useBloomFilterCol10 =
PropertyUtil.propertyAsBoolean(
table.properties(),
PARQUET_BLOOM_FILTER_COLUMN_ENABLED_PREFIX + "id_fixed_decimal",
false);
- factory.set(
+ writeProperties.put(
PARQUET_BLOOM_FILTER_COLUMN_ENABLED_PREFIX + "id_fixed_decimal",
Boolean.toString(useBloomFilterCol10));
int blockSize =
PropertyUtil.propertyAsInt(
table.properties(), PARQUET_ROW_GROUP_SIZE_BYTES,
PARQUET_ROW_GROUP_SIZE_BYTES_DEFAULT);
- factory.set(PARQUET_ROW_GROUP_SIZE_BYTES, Integer.toString(blockSize));
-
- FileAppender<Record> writer = factory.newAppender(out, format);
- try (Closeable toClose = writer) {
- writer.addAll(rows);
+ writeProperties.put(PARQUET_ROW_GROUP_SIZE_BYTES,
Integer.toString(blockSize));
+
+ DataWriter<Record> writer =
+ new GenericFileWriterFactory.Builder()
+ .dataSchema(table.schema())
+ .dataFileFormat(format)
+ .writerProperties(writeProperties)
Review Comment:
The sad truth is that this test is not checking if the bloom filters are
actually configured/created or not.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
