huaxingao commented on code in PR #4831: URL: https://github.com/apache/iceberg/pull/4831#discussion_r878799822
########## spark/v3.2/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderWithBloomFilter.java: ########## @@ -0,0 +1,248 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iceberg.spark.source; + +import java.io.IOException; +import java.math.BigDecimal; +import java.time.Instant; +import java.time.LocalDate; +import java.time.ZoneOffset; +import java.util.List; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.iceberg.BaseTable; +import org.apache.iceberg.CatalogUtil; +import org.apache.iceberg.DataFile; +import org.apache.iceberg.Files; +import org.apache.iceberg.Schema; +import org.apache.iceberg.Table; +import org.apache.iceberg.TableMetadata; +import org.apache.iceberg.TableOperations; +import org.apache.iceberg.TableProperties; +import org.apache.iceberg.TestHelpers.Row; +import org.apache.iceberg.catalog.Namespace; +import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.iceberg.data.FileHelpers; +import org.apache.iceberg.data.GenericRecord; +import org.apache.iceberg.data.Record; +import org.apache.iceberg.exceptions.AlreadyExistsException; +import org.apache.iceberg.hive.HiveCatalog; +import org.apache.iceberg.hive.TestHiveMetastore; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; +import org.apache.iceberg.relocated.com.google.common.collect.Lists; +import org.apache.iceberg.spark.SparkValueConverter; +import org.apache.iceberg.types.Types; +import org.apache.spark.sql.Dataset; +import org.apache.spark.sql.SparkSession; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +import static org.apache.hadoop.hive.conf.HiveConf.ConfVars.METASTOREURIS; + +@RunWith(Parameterized.class) +public class TestSparkReaderWithBloomFilter { + + protected String tableName = null; + protected Table table = null; + protected List<Record> records = null; + protected DataFile dataFile = null; + + private static TestHiveMetastore metastore = null; + protected static SparkSession spark = null; + protected static HiveCatalog catalog = null; + protected final boolean vectorized; + protected final boolean useBloomFilter; + + public TestSparkReaderWithBloomFilter(boolean vectorized, boolean useBloomFilter) { + this.vectorized = vectorized; + this.useBloomFilter = useBloomFilter; + } + + // Schema passed to create tables + public static final Schema SCHEMA = new Schema( + Types.NestedField.required(1, "id", Types.IntegerType.get()), + Types.NestedField.required(2, "id_long", Types.LongType.get()), + Types.NestedField.required(3, "id_double", Types.DoubleType.get()), + Types.NestedField.required(4, "id_float", Types.FloatType.get()), + Types.NestedField.required(5, "id_string", Types.StringType.get()), + Types.NestedField.optional(6, "id_boolean", Types.BooleanType.get()), + Types.NestedField.optional(7, "id_date", Types.DateType.get()), + Types.NestedField.optional(8, "id_timestamp", Types.TimestampType.withZone()), + Types.NestedField.optional(9, "id_int_decimal", Types.DecimalType.of(8, 2)), + Types.NestedField.optional(10, "id_long_decimal", Types.DecimalType.of(14, 2)), + Types.NestedField.optional(11, "id_fixed_decimal", Types.DecimalType.of(31, 2)) + ); + + private static final int INT_MIN_VALUE = 30; + private static final int INT_MAX_VALUE = 329; + private static final int INT_VALUE_COUNT = INT_MAX_VALUE - INT_MIN_VALUE + 1; + private static final long LONG_BASE = 1000L; + private static final double DOUBLE_BASE = 10000D; + private static final float FLOAT_BASE = 100000F; + private static final String BINARY_PREFIX = "BINARY测试_"; + + @Rule + public TemporaryFolder temp = new TemporaryFolder(); + + @Before + public void writeTestDataFile() throws IOException { + this.tableName = "test"; + createTable(tableName, SCHEMA); + this.records = Lists.newArrayList(); + + // records all use IDs that are in bucket id_bucket=0 + GenericRecord record = GenericRecord.create(table.schema()); + + + for (int i = 0; i < INT_VALUE_COUNT; i += 1) { + records.add(record.copy( + "id", INT_MIN_VALUE + i, + "id_long", LONG_BASE + INT_MIN_VALUE + i, + "id_double", DOUBLE_BASE + INT_MIN_VALUE + i, + "id_float", FLOAT_BASE + INT_MIN_VALUE + i, + "id_string", BINARY_PREFIX + (INT_MIN_VALUE + i), + "id_boolean", (i % 2 == 0) ? true : false, + "id_date", LocalDate.parse("2021-09-05"), + "id_timestamp", Instant.ofEpochMilli(0L).atOffset(ZoneOffset.UTC), + "id_int_decimal", new BigDecimal(String.valueOf(77.77)), + "id_long_decimal", new BigDecimal(String.valueOf(88.88)), + "id_fixed_decimal", new BigDecimal(String.valueOf(99.99)))); + } + + this.dataFile = FileHelpers.writeDataFile(table, Files.localOutput(temp.newFile()), Row.of(0), records); + + table.newAppend() + .appendFile(dataFile) + .commit(); + } + + @After + public void cleanup() throws IOException { + dropTable("test"); + } + + @Parameterized.Parameters(name = "vectorized = {0}, useBloomFilter = {1}") + public static Object[][] parameters() { + return new Object[][] { + {false, false}, {true, false}, {false, true}, {true, true} + }; + } + + @BeforeClass + public static void startMetastoreAndSpark() { + metastore = new TestHiveMetastore(); + metastore.start(); + HiveConf hiveConf = metastore.hiveConf(); + + spark = SparkSession.builder() + .master("local[2]") + .config("spark.hadoop." + METASTOREURIS.varname, hiveConf.get(METASTOREURIS.varname)) + .enableHiveSupport() + .getOrCreate(); + + catalog = (HiveCatalog) + CatalogUtil.loadCatalog(HiveCatalog.class.getName(), "hive", ImmutableMap.of(), hiveConf); + + try { + catalog.createNamespace(Namespace.of("default")); + } catch (AlreadyExistsException ignored) { + // the default namespace already exists. ignore the create error + } + } + + @AfterClass + public static void stopMetastoreAndSpark() throws Exception { + catalog = null; + metastore.stop(); + metastore = null; + spark.stop(); + spark = null; + } + + protected void createTable(String name, Schema schema) { + table = catalog.createTable(TableIdentifier.of("default", name), schema); + TableOperations ops = ((BaseTable) table).operations(); + TableMetadata meta = ops.current(); + ops.commit(meta, meta.upgradeToFormatVersion(2)); + + if (useBloomFilter) { + table.updateProperties() + .set(TableProperties.DEFAULT_PARQUET_BLOOM_FILTER_ENABLED, String.valueOf(useBloomFilter)) + .commit(); + } + + table.updateProperties() + .set(TableProperties.PARQUET_ROW_GROUP_SIZE_BYTES, "100") // to have multiple row groups + .commit(); + if (vectorized) { + table.updateProperties() + .set(TableProperties.PARQUET_VECTORIZATION_ENABLED, "true") + .set(TableProperties.PARQUET_BATCH_SIZE, "4") + .commit(); + } Review Comment: These properties can be set either at table creation or updated later. I don't think it matters. The reason I picked the second choice is because when I wrote the test, I happened to use `TestSparkReaderDeletes` as my template and followed the style there. ########## parquet/src/test/java/org/apache/iceberg/parquet/TestBloomRowGroupFilter.java: ########## @@ -0,0 +1,950 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iceberg.parquet; + +import java.io.File; +import java.io.IOException; +import java.math.BigDecimal; +import java.time.Instant; +import java.util.List; +import java.util.Random; +import java.util.UUID; +import java.util.stream.Collectors; +import java.util.stream.IntStream; +import org.apache.avro.generic.GenericData.Record; +import org.apache.avro.generic.GenericRecordBuilder; +import org.apache.iceberg.Files; +import org.apache.iceberg.Schema; +import org.apache.iceberg.TestHelpers; +import org.apache.iceberg.avro.AvroSchemaUtil; +import org.apache.iceberg.exceptions.ValidationException; +import org.apache.iceberg.expressions.Expression; +import org.apache.iceberg.io.FileAppender; +import org.apache.iceberg.io.InputFile; +import org.apache.iceberg.io.OutputFile; +import org.apache.iceberg.relocated.com.google.common.collect.Lists; +import org.apache.iceberg.types.Types; +import org.apache.iceberg.types.Types.DoubleType; +import org.apache.iceberg.types.Types.FloatType; +import org.apache.iceberg.types.Types.IntegerType; +import org.apache.iceberg.types.Types.LongType; +import org.apache.iceberg.types.Types.StringType; +import org.apache.iceberg.types.Types.UUIDType; +import org.apache.parquet.column.values.bloomfilter.BloomFilter; +import org.apache.parquet.hadoop.BloomFilterReader; +import org.apache.parquet.hadoop.ParquetFileReader; +import org.apache.parquet.hadoop.metadata.BlockMetaData; +import org.apache.parquet.hadoop.metadata.ColumnChunkMetaData; +import org.apache.parquet.schema.MessageType; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import static org.apache.iceberg.TableProperties.DEFAULT_PARQUET_BLOOM_FILTER_ENABLED; +import static org.apache.iceberg.TableProperties.PARQUET_BLOOM_FILTER_COLUMN_ENABLED_PREFIX; +import static org.apache.iceberg.avro.AvroSchemaUtil.convert; +import static org.apache.iceberg.expressions.Expressions.and; +import static org.apache.iceberg.expressions.Expressions.equal; +import static org.apache.iceberg.expressions.Expressions.greaterThan; +import static org.apache.iceberg.expressions.Expressions.greaterThanOrEqual; +import static org.apache.iceberg.expressions.Expressions.in; +import static org.apache.iceberg.expressions.Expressions.isNaN; +import static org.apache.iceberg.expressions.Expressions.isNull; +import static org.apache.iceberg.expressions.Expressions.lessThan; +import static org.apache.iceberg.expressions.Expressions.lessThanOrEqual; +import static org.apache.iceberg.expressions.Expressions.not; +import static org.apache.iceberg.expressions.Expressions.notEqual; +import static org.apache.iceberg.expressions.Expressions.notIn; +import static org.apache.iceberg.expressions.Expressions.notNaN; +import static org.apache.iceberg.expressions.Expressions.notNull; +import static org.apache.iceberg.expressions.Expressions.or; +import static org.apache.iceberg.expressions.Expressions.startsWith; +import static org.apache.iceberg.types.Types.NestedField.optional; +import static org.apache.iceberg.types.Types.NestedField.required; + +public class TestBloomRowGroupFilter { + + private static final Types.StructType structFieldType = + Types.StructType.of(Types.NestedField.required(16, "int_field", IntegerType.get())); + private static final Schema SCHEMA = new Schema( + required(1, "id", IntegerType.get()), + required(2, "long", LongType.get()), + required(3, "double", DoubleType.get()), + required(4, "float", FloatType.get()), + required(5, "string", StringType.get()), + required(6, "uuid", UUIDType.get()), + required(7, "required", StringType.get()), + optional(8, "non_bloom", StringType.get()), + optional(9, "all_nulls", LongType.get()), + optional(10, "some_nulls", StringType.get()), + optional(11, "no_nulls", StringType.get()), + optional(12, "all_nans", DoubleType.get()), + optional(13, "some_nans", FloatType.get()), + optional(14, "no_nans", DoubleType.get()), + optional(15, "struct_not_null", structFieldType), + optional(17, "not_in_file", FloatType.get()), + optional(18, "no_stats", StringType.get()), + optional(19, "boolean", Types.BooleanType.get()), + optional(20, "time", Types.TimeType.get()), + optional(21, "date", Types.DateType.get()), + optional(22, "timestamp", Types.TimestampType.withoutZone()), + optional(23, "timestamptz", Types.TimestampType.withZone()), + optional(24, "binary", Types.BinaryType.get()), + optional(25, "int_decimal", Types.DecimalType.of(8, 2)), + optional(26, "long_decimal", Types.DecimalType.of(14, 2)), + optional(27, "fixed_decimal", Types.DecimalType.of(31, 2)) + ); + + private static final Types.StructType _structFieldType = + Types.StructType.of(Types.NestedField.required(16, "_int_field", IntegerType.get())); + + private static final Schema FILE_SCHEMA = new Schema( + required(1, "_id", IntegerType.get()), + required(2, "_long", LongType.get()), + required(3, "_double", DoubleType.get()), + required(4, "_float", FloatType.get()), + required(5, "_string", StringType.get()), + required(6, "_uuid", UUIDType.get()), + required(7, "_required", StringType.get()), + required(8, "_non_bloom", StringType.get()), + optional(9, "_all_nulls", LongType.get()), + optional(10, "_some_nulls", StringType.get()), + optional(11, "_no_nulls", StringType.get()), + optional(12, "_all_nans", DoubleType.get()), + optional(13, "_some_nans", FloatType.get()), + optional(14, "_no_nans", DoubleType.get()), + optional(15, "_struct_not_null", _structFieldType), + optional(18, "_no_stats", StringType.get()), + optional(19, "_boolean", Types.BooleanType.get()), + optional(20, "_time", Types.TimeType.get()), + optional(21, "_date", Types.DateType.get()), + optional(22, "_timestamp", Types.TimestampType.withoutZone()), + optional(23, "_timestamptz", Types.TimestampType.withZone()), + optional(24, "_binary", Types.BinaryType.get()), + optional(25, "_int_decimal", Types.DecimalType.of(8, 2)), + optional(26, "_long_decimal", Types.DecimalType.of(14, 2)), + optional(27, "_fixed_decimal", Types.DecimalType.of(31, 2)) + ); + + private static final String TOO_LONG_FOR_STATS; + + static { + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < 200; i += 1) { + sb.append(UUID.randomUUID().toString()); + } + TOO_LONG_FOR_STATS = sb.toString(); + } + + private static final int INT_MIN_VALUE = 30; + private static final int INT_MAX_VALUE = 79; + private static final int INT_VALUE_COUNT = INT_MAX_VALUE - INT_MIN_VALUE + 1; + private static final long LONG_BASE = 100L; + private static final double DOUBLE_BASE = 1000D; + private static final float FLOAT_BASE = 10000F; + private static final String BINARY_PREFIX = "BINARY测试_"; + private static final Instant instant = Instant.parse("2018-10-10T00:00:00.000Z"); + private static final List<UUID> RANDOM_UUIDS; + private static final List<byte[]> RANDOM_BYTES; + + static { + RANDOM_UUIDS = Lists.newArrayList(); + for (int i = 0; i < INT_VALUE_COUNT; i += 1) { + RANDOM_UUIDS.add(UUID.randomUUID()); + } + + RANDOM_BYTES = Lists.newArrayList(); + Random rd = new Random(); + for (int i = 1; i <= INT_VALUE_COUNT; i += 1) { + byte[] byteArray = new byte[i]; + rd.nextBytes(byteArray); + RANDOM_BYTES.add(byteArray); + } + } + + private MessageType parquetSchema = null; + private BlockMetaData rowGroupMetadata = null; + private BloomFilterReader bloomStore = null; + + @Rule + public TemporaryFolder temp = new TemporaryFolder(); + + @Before + public void createInputFile() throws IOException { + File parquetFile = temp.newFile(); + Assert.assertTrue(parquetFile.delete()); + + // build struct field schema + org.apache.avro.Schema structSchema = AvroSchemaUtil.convert(_structFieldType); + + OutputFile outFile = Files.localOutput(parquetFile); + try (FileAppender<Record> appender = Parquet.write(outFile) + .schema(FILE_SCHEMA) + .set(DEFAULT_PARQUET_BLOOM_FILTER_ENABLED, "true") + .set(PARQUET_BLOOM_FILTER_COLUMN_ENABLED_PREFIX + "_non_bloom", "false") + .build()) { + GenericRecordBuilder builder = new GenericRecordBuilder(convert(FILE_SCHEMA, "table")); + // create 50 records + for (int i = 0; i < INT_VALUE_COUNT; i += 1) { + builder.set("_id", INT_MIN_VALUE + i); // min=30, max=79, num-nulls=0 + builder.set("_long", LONG_BASE + INT_MIN_VALUE + i); // min=130L, max=179L, num-nulls=0 + builder.set("_double", DOUBLE_BASE + INT_MIN_VALUE + i); // min=1030D, max=1079D, num-nulls=0 + builder.set("_float", FLOAT_BASE + INT_MIN_VALUE + i); // min=10030F, max=10079F, num-nulls=0 + builder.set("_string", BINARY_PREFIX + (INT_MIN_VALUE + i)); // min=BINARY测试_30, max=BINARY测试_79, num-nulls=0 + builder.set("_uuid", RANDOM_UUIDS.get(i)); // required, random uuid, always non-null + builder.set("_required", "req"); // required, always non-null + builder.set("_non_bloom", RANDOM_UUIDS.get(i)); // bloom filter not enabled + builder.set("_all_nulls", null); // never non-null + builder.set("_some_nulls", (i % 10 == 0) ? null : "some"); // includes some null values + builder.set("_no_nulls", ""); // optional, but always non-null + builder.set("_all_nans", Double.NaN); // never non-nan + builder.set("_some_nans", (i % 10 == 0) ? Float.NaN : 2F); // includes some nan values + builder.set("_no_nans", 3D); // optional, but always non-nan + Record structNotNull = new Record(structSchema); + structNotNull.put("_int_field", INT_MIN_VALUE + i); + builder.set("_struct_not_null", structNotNull); // struct with int + builder.set("_no_stats", TOO_LONG_FOR_STATS); // value longer than 4k will produce no stats + builder.set("_boolean", (i % 2 == 0) ? true : false); + builder.set("_time", instant.plusSeconds(i * 86400).toEpochMilli()); + builder.set("_date", instant.plusSeconds(i * 86400).getEpochSecond()); + builder.set("_timestamp", instant.plusSeconds(i * 86400).toEpochMilli()); + builder.set("_timestamptz", instant.plusSeconds(i * 86400).toEpochMilli()); + builder.set("_binary", RANDOM_BYTES.get(i)); + builder.set("_int_decimal", new BigDecimal(String.valueOf(77.77 + i))); + builder.set("_long_decimal", new BigDecimal(String.valueOf(88.88 + i))); + builder.set("_fixed_decimal", new BigDecimal(String.valueOf(99.99 + i))); + + appender.add(builder.build()); + } + } + + InputFile inFile = Files.localInput(parquetFile); + + ParquetFileReader reader = ParquetFileReader.open(ParquetIO.file(inFile)); + + Assert.assertEquals("Should create only one row group", 1, reader.getRowGroups().size()); + rowGroupMetadata = reader.getRowGroups().get(0); + parquetSchema = reader.getFileMetaData().getSchema(); + bloomStore = reader.getBloomFilterDataReader(rowGroupMetadata); + } + + @Test + public void testNotNull() { + boolean shouldRead = new ParquetBloomRowGroupFilter(SCHEMA, notNull("all_nulls")) + .shouldRead(parquetSchema, rowGroupMetadata, bloomStore); + Assert.assertTrue("Should read: bloom filter doesn't help", shouldRead); + + shouldRead = new ParquetBloomRowGroupFilter(SCHEMA, notNull("some_nulls")) + .shouldRead(parquetSchema, rowGroupMetadata, bloomStore); + Assert.assertTrue("Should read: bloom filter doesn't help", shouldRead); + + shouldRead = new ParquetBloomRowGroupFilter(SCHEMA, notNull("no_nulls")) + .shouldRead(parquetSchema, rowGroupMetadata, bloomStore); + Assert.assertTrue("Should read: bloom filter doesn't help", shouldRead); + + shouldRead = new ParquetBloomRowGroupFilter(SCHEMA, notNull("struct_not_null")) + .shouldRead(parquetSchema, rowGroupMetadata, bloomStore); + Assert.assertTrue("Should read: bloom filter doesn't help", shouldRead); + } + + @Test + public void testIsNull() { + boolean shouldRead = new ParquetBloomRowGroupFilter(SCHEMA, isNull("all_nulls")) + .shouldRead(parquetSchema, rowGroupMetadata, bloomStore); + Assert.assertTrue("Should read: bloom filter doesn't help", shouldRead); + + shouldRead = new ParquetBloomRowGroupFilter(SCHEMA, isNull("some_nulls")) + .shouldRead(parquetSchema, rowGroupMetadata, bloomStore); + Assert.assertTrue("Should read: bloom filter doesn't help", shouldRead); + + shouldRead = new ParquetBloomRowGroupFilter(SCHEMA, isNull("no_nulls")) + .shouldRead(parquetSchema, rowGroupMetadata, bloomStore); + Assert.assertTrue("Should read: bloom filter doesn't help", shouldRead); + + shouldRead = new ParquetBloomRowGroupFilter(SCHEMA, isNull("struct_not_null")) + .shouldRead(parquetSchema, rowGroupMetadata, bloomStore); + Assert.assertTrue("Should read: bloom filter doesn't help", shouldRead); + } + + @Test + public void testRequiredColumn() { + boolean shouldRead = new ParquetBloomRowGroupFilter(SCHEMA, notNull("required")) + .shouldRead(parquetSchema, rowGroupMetadata, bloomStore); + Assert.assertTrue("Should read: required columns are always non-null", shouldRead); + + shouldRead = new ParquetBloomRowGroupFilter(SCHEMA, isNull("required")) + .shouldRead(parquetSchema, rowGroupMetadata, bloomStore); + Assert.assertFalse("Should skip: required columns are always non-null", shouldRead); + } + + @Test + public void testIsNaNs() { + boolean shouldRead = new ParquetBloomRowGroupFilter(SCHEMA, isNaN("all_nans")) + .shouldRead(parquetSchema, rowGroupMetadata, bloomStore); + Assert.assertTrue("Should read: bloom filter doesn't help", shouldRead); + + shouldRead = new ParquetBloomRowGroupFilter(SCHEMA, isNaN("some_nans")) + .shouldRead(parquetSchema, rowGroupMetadata, bloomStore); + Assert.assertTrue("Should read: bloom filter doesn't help", shouldRead); + + shouldRead = new ParquetBloomRowGroupFilter(SCHEMA, isNaN("no_nans")) + .shouldRead(parquetSchema, rowGroupMetadata, bloomStore); + Assert.assertTrue("Should read: bloom filter doesn't help", shouldRead); + } + + @Test + public void testNotNaNs() { + boolean shouldRead = new ParquetBloomRowGroupFilter(SCHEMA, notNaN("all_nans")) + .shouldRead(parquetSchema, rowGroupMetadata, bloomStore); + Assert.assertTrue("Should read: bloom filter doesn't help", shouldRead); + + shouldRead = new ParquetBloomRowGroupFilter(SCHEMA, notNaN("some_nans")) + .shouldRead(parquetSchema, rowGroupMetadata, bloomStore); + Assert.assertTrue("Should read: bloom filter doesn't help", shouldRead); + + shouldRead = new ParquetBloomRowGroupFilter(SCHEMA, notNaN("no_nans")) + .shouldRead(parquetSchema, rowGroupMetadata, bloomStore); + Assert.assertTrue("Should read: bloom filter doesn't help", shouldRead); + } + + @Test + public void testStartsWith() { + boolean shouldRead = new ParquetBloomRowGroupFilter(SCHEMA, startsWith("non_bloom", "re")) + .shouldRead(parquetSchema, rowGroupMetadata, bloomStore); + Assert.assertTrue("Should read: no bloom", shouldRead); + + shouldRead = new ParquetBloomRowGroupFilter(SCHEMA, startsWith("required", "re")) + .shouldRead(parquetSchema, rowGroupMetadata, bloomStore); + Assert.assertTrue("Should read: bloom filter doesn't help", shouldRead); + + shouldRead = new ParquetBloomRowGroupFilter(SCHEMA, startsWith("required", "req")) + .shouldRead(parquetSchema, rowGroupMetadata, bloomStore); + Assert.assertTrue("Should read: bloom filter doesn't help", shouldRead); + + shouldRead = new ParquetBloomRowGroupFilter(SCHEMA, startsWith("some_nulls", "so")) + .shouldRead(parquetSchema, rowGroupMetadata, bloomStore); + Assert.assertTrue("Should read: bloom filter doesn't help", shouldRead); + + shouldRead = new ParquetBloomRowGroupFilter(SCHEMA, startsWith("required", "reqs")) + .shouldRead(parquetSchema, rowGroupMetadata, bloomStore); + Assert.assertTrue("Should read: bloom filter doesn't help", shouldRead); + + shouldRead = new ParquetBloomRowGroupFilter(SCHEMA, startsWith("some_nulls", "somex")) + .shouldRead(parquetSchema, rowGroupMetadata, bloomStore); + Assert.assertTrue("Should read: bloom filter doesn't help", shouldRead); + + shouldRead = new ParquetBloomRowGroupFilter(SCHEMA, startsWith("no_nulls", "xxx")) + .shouldRead(parquetSchema, rowGroupMetadata, bloomStore); + Assert.assertTrue("Should read: bloom filter doesn't help", shouldRead); + } + + @Test + public void testMissingColumn() { + TestHelpers.assertThrows("Should complain about missing column in expression", + ValidationException.class, "Cannot find field 'missing'", + () -> new ParquetBloomRowGroupFilter(SCHEMA, lessThan("missing", 5)) + .shouldRead(parquetSchema, rowGroupMetadata, bloomStore)); + } + + @Test + public void testColumnNotInFile() { + Expression[] exprs = new Expression[]{ + lessThan("not_in_file", 1.0f), lessThanOrEqual("not_in_file", 1.0f), + equal("not_in_file", 1.0f), greaterThan("not_in_file", 1.0f), + greaterThanOrEqual("not_in_file", 1.0f), notNull("not_in_file"), + isNull("not_in_file"), notEqual("not_in_file", 1.0f), in("not_in_file", 1.0f, 2.0f) + }; + + for (Expression expr : exprs) { + boolean shouldRead = new ParquetBloomRowGroupFilter(SCHEMA, expr) + .shouldRead(parquetSchema, rowGroupMetadata, bloomStore); + Assert.assertTrue("Should read: bloom filter cannot be found: " + expr, shouldRead); + } + } + + @Test + public void testColumnNotBloomFilterEnabled() { + Expression[] exprs = new Expression[]{ + lessThan("non_bloom", "a"), lessThanOrEqual("non_bloom", "a"), equal("non_bloom", "a"), + greaterThan("non_bloom", "a"), greaterThanOrEqual("non_bloom", "a"), notNull("non_bloom"), + isNull("non_bloom"), notEqual("non_bloom", "a") + }; + + for (Expression expr : exprs) { + boolean shouldRead = new ParquetBloomRowGroupFilter(SCHEMA, expr) + .shouldRead(parquetSchema, rowGroupMetadata, bloomStore); + Assert.assertTrue("Should read: bloom filter cannot be found: " + expr, shouldRead); + } + } + + @Test + public void testMissingStats() { + boolean shouldRead = new ParquetBloomRowGroupFilter(SCHEMA, equal("no_stats", "a")) + .shouldRead(parquetSchema, rowGroupMetadata, bloomStore); + Assert.assertFalse("Should skip: stats are missing but bloom filter is present", shouldRead); + } + + @Test + public void testNot() { + // this test case must use a real predicate, not alwaysTrue(), otherwise binding will simplify it out + for (int i = INT_MIN_VALUE - 20; i < INT_MAX_VALUE + 20; i++) { + boolean shouldRead = new ParquetBloomRowGroupFilter(SCHEMA, not(equal("id", i))) Review Comment: Fixed. Thanks -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
