This is an automated email from the ASF dual-hosted git repository.
nehapawar pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new b837350034 Add minMaxInvalid flag to avoid unnecessary needPreprocess
(#9238)
b837350034 is described below
commit b837350034dfb9dc5d8ea624b1e11bbbef3b4948
Author: Neha Pawar <[email protected]>
AuthorDate: Tue Aug 30 22:49:19 2022 -0700
Add minMaxInvalid flag to avoid unnecessary needPreprocess (#9238)
* Add minMaxInvalid flag to avoid unecessary needPreprocess
* Increase size to account for new field in metadata
---
.../tests/OfflineClusterIntegrationTest.java | 2 +-
.../creator/impl/SegmentColumnarIndexCreator.java | 4 +
.../ColumnMinMaxValueGenerator.java | 2 +-
.../impl/SegmentColumnarIndexCreatorTest.java | 24 ++++-
.../SegmentGenerationWithMinMaxInvalidTest.java | 105 +++++++++++++++++++++
.../index/loader/SegmentPreProcessorTest.java | 77 ++++++++++++++-
.../apache/pinot/segment/spi/ColumnMetadata.java | 4 +
.../org/apache/pinot/segment/spi/V1Constants.java | 1 +
.../spi/index/metadata/ColumnMetadataImpl.java | 27 ++++--
9 files changed, 233 insertions(+), 13 deletions(-)
diff --git
a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/OfflineClusterIntegrationTest.java
b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/OfflineClusterIntegrationTest.java
index 0e0a79c4a9..be4cd7c3db 100644
---
a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/OfflineClusterIntegrationTest.java
+++
b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/OfflineClusterIntegrationTest.java
@@ -143,7 +143,7 @@ public class OfflineClusterIntegrationTest extends
BaseClusterIntegrationTestSet
private static final String MAX_NUM_MULTI_VALUES_MAP_KEY =
"maxNumMultiValuesMap";
// TODO: This might lead to flaky test, as this disk size is not
deterministic
// as it depends on the iteration order of a HashSet.
- private static final int DISK_SIZE_IN_BYTES = 20796000;
+ private static final int DISK_SIZE_IN_BYTES = 20797128;
private static final int NUM_ROWS = 115545;
private final List<ServiceStatus.ServiceStatusCallback>
_serviceStatusCallbacks =
diff --git
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/SegmentColumnarIndexCreator.java
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/SegmentColumnarIndexCreator.java
index 7357d8bd05..7de25bcebc 100644
---
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/SegmentColumnarIndexCreator.java
+++
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/SegmentColumnarIndexCreator.java
@@ -856,9 +856,13 @@ public class SegmentColumnarIndexCreator implements
SegmentCreator {
String maxValue) {
if (isValidPropertyValue(minValue)) {
properties.setProperty(getKeyFor(column, MIN_VALUE), minValue);
+ } else {
+ properties.setProperty(getKeyFor(column, MIN_MAX_VALUE_INVALID), true);
}
if (isValidPropertyValue(maxValue)) {
properties.setProperty(getKeyFor(column, MAX_VALUE), maxValue);
+ } else {
+ properties.setProperty(getKeyFor(column, MIN_MAX_VALUE_INVALID), true);
}
}
diff --git
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/columnminmaxvalue/ColumnMinMaxValueGenerator.java
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/columnminmaxvalue/ColumnMinMaxValueGenerator.java
index 77351894da..1a06f686cd 100644
---
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/columnminmaxvalue/ColumnMinMaxValueGenerator.java
+++
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/columnminmaxvalue/ColumnMinMaxValueGenerator.java
@@ -98,7 +98,7 @@ public class ColumnMinMaxValueGenerator {
private boolean needAddColumnMinMaxValueForColumn(String columnName) {
ColumnMetadata columnMetadata =
_segmentMetadata.getColumnMetadataFor(columnName);
return columnMetadata.hasDictionary() && columnMetadata.getMinValue() ==
null
- && columnMetadata.getMaxValue() == null;
+ && columnMetadata.getMaxValue() == null &&
!columnMetadata.isMinMaxValueInvalid();
}
private void addColumnMinMaxValueForColumn(String columnName)
diff --git
a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/creator/impl/SegmentColumnarIndexCreatorTest.java
b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/creator/impl/SegmentColumnarIndexCreatorTest.java
index 49ff6cf35d..7b4dc27deb 100644
---
a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/creator/impl/SegmentColumnarIndexCreatorTest.java
+++
b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/creator/impl/SegmentColumnarIndexCreatorTest.java
@@ -29,7 +29,7 @@ import
org.apache.pinot.segment.local.indexsegment.immutable.ImmutableSegmentLoa
import org.apache.pinot.segment.local.segment.readers.GenericRowRecordReader;
import org.apache.pinot.segment.spi.IndexSegment;
import org.apache.pinot.segment.spi.SegmentMetadata;
-import org.apache.pinot.segment.spi.V1Constants;
+import org.apache.pinot.segment.spi.V1Constants.MetadataKeys.Column;
import org.apache.pinot.segment.spi.creator.SegmentGeneratorConfig;
import org.apache.pinot.spi.config.table.TableConfig;
import org.apache.pinot.spi.config.table.TableType;
@@ -43,14 +43,12 @@ import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;
-
public class SegmentColumnarIndexCreatorTest {
private static final File TEMP_DIR = new File(FileUtils.getTempDirectory(),
"SegmentColumnarIndexCreatorTest");
private static final File CONFIG_FILE = new File(TEMP_DIR, "config");
private static final String PROPERTY_KEY = "testKey";
private static final String COLUMN_NAME = "testColumn";
- private static final String COLUMN_PROPERTY_KEY_PREFIX =
- V1Constants.MetadataKeys.Column.COLUMN_PROPS_KEY_PREFIX + COLUMN_NAME +
".";
+ private static final String COLUMN_PROPERTY_KEY_PREFIX =
Column.COLUMN_PROPS_KEY_PREFIX + COLUMN_NAME + ".";
private static final int NUM_ROUNDS = 1000;
@BeforeClass
@@ -175,6 +173,24 @@ public class SegmentColumnarIndexCreatorTest {
}
}
+ @Test
+ public void testAddMinMaxValueInvalid() {
+ PropertiesConfiguration props = new PropertiesConfiguration();
+ SegmentColumnarIndexCreator.addColumnMinMaxValueInfo(props, "colA", "bar",
"foo");
+ Assert.assertFalse(Boolean.parseBoolean(
+ String.valueOf(props.getProperty(Column.getKeyFor("colA",
Column.MIN_MAX_VALUE_INVALID)))));
+
+ props = new PropertiesConfiguration();
+ SegmentColumnarIndexCreator.addColumnMinMaxValueInfo(props, "colA",
",bar", "foo");
+ Assert.assertTrue(Boolean.parseBoolean(
+ String.valueOf(props.getProperty(Column.getKeyFor("colA",
Column.MIN_MAX_VALUE_INVALID)))));
+
+ props = new PropertiesConfiguration();
+ SegmentColumnarIndexCreator.addColumnMinMaxValueInfo(props, "colA", "bar",
" ");
+ Assert.assertTrue(Boolean.parseBoolean(
+ String.valueOf(props.getProperty(Column.getKeyFor("colA",
Column.MIN_MAX_VALUE_INVALID)))));
+ }
+
@AfterClass
public void tearDown()
throws IOException {
diff --git
a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/creator/SegmentGenerationWithMinMaxInvalidTest.java
b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/creator/SegmentGenerationWithMinMaxInvalidTest.java
new file mode 100644
index 0000000000..9ed3062931
--- /dev/null
+++
b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/creator/SegmentGenerationWithMinMaxInvalidTest.java
@@ -0,0 +1,105 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.segment.local.segment.index.creator;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.commons.io.FileUtils;
+import
org.apache.pinot.segment.local.segment.creator.impl.SegmentIndexCreationDriverImpl;
+import org.apache.pinot.segment.local.segment.readers.GenericRowRecordReader;
+import org.apache.pinot.segment.spi.creator.SegmentGeneratorConfig;
+import org.apache.pinot.segment.spi.index.metadata.SegmentMetadataImpl;
+import org.apache.pinot.spi.config.table.TableConfig;
+import org.apache.pinot.spi.config.table.TableType;
+import org.apache.pinot.spi.data.FieldSpec;
+import org.apache.pinot.spi.data.Schema;
+import org.apache.pinot.spi.data.readers.GenericRow;
+import org.apache.pinot.spi.utils.builder.TableConfigBuilder;
+import org.testng.Assert;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Test;
+
+
+/**
+ * Tests filtering of records during segment generation
+ */
+public class SegmentGenerationWithMinMaxInvalidTest {
+ private static final String STRING_COLUMN = "col1";
+ private static final String[] STRING_VALUES_INVALID = {"A,", "B,", "C,",
"D,", "E"};
+ private static final String[] STRING_VALUES_VALID = {"A", "B", "C", "D",
"E"};
+ private static final String LONG_COLUMN = "col2";
+ private static final long[] LONG_VALUES =
+ {1588316400000L, 1588489200000L, 1588662000000L, 1588834800000L,
1589007600000L};
+
+ private static final String SEGMENT_DIR_NAME =
+ FileUtils.getTempDirectoryPath() + File.separator +
"segmentMinMaxInvalidTest";
+ private static final String SEGMENT_NAME = "testSegmentMinMaxInvalid";
+
+ private Schema _schema;
+ private TableConfig _tableConfig;
+
+ @BeforeClass
+ public void setup() {
+ _tableConfig = new
TableConfigBuilder(TableType.OFFLINE).setTableName("testTable").build();
+ _schema = new
Schema.SchemaBuilder().addSingleValueDimension(STRING_COLUMN,
FieldSpec.DataType.STRING)
+ .addMetric(LONG_COLUMN, FieldSpec.DataType.LONG).build();
+ }
+
+ @Test
+ public void testMinMaxInvalidFlagInMetadata()
+ throws Exception {
+ FileUtils.deleteQuietly(new File(SEGMENT_DIR_NAME));
+ File segmentDir = buildSegment(_tableConfig, _schema, STRING_VALUES_VALID);
+ SegmentMetadataImpl metadata = new SegmentMetadataImpl(segmentDir);
+ Assert.assertEquals(metadata.getTotalDocs(), 5);
+
Assert.assertFalse(metadata.getColumnMetadataFor("col1").isMinMaxValueInvalid());
+ Assert.assertEquals(metadata.getColumnMetadataFor("col1").getMinValue(),
"A");
+ Assert.assertEquals(metadata.getColumnMetadataFor("col1").getMaxValue(),
"E");
+
+ FileUtils.deleteQuietly(new File(SEGMENT_DIR_NAME));
+ segmentDir = buildSegment(_tableConfig, _schema, STRING_VALUES_INVALID);
+ metadata = new SegmentMetadataImpl(segmentDir);
+ Assert.assertEquals(metadata.getTotalDocs(), 5);
+
Assert.assertTrue(metadata.getColumnMetadataFor("col1").isMinMaxValueInvalid());
+ Assert.assertNull(metadata.getColumnMetadataFor("col1").getMinValue());
+ Assert.assertNull(metadata.getColumnMetadataFor("col1").getMaxValue());
+ }
+
+ private File buildSegment(final TableConfig tableConfig, final Schema
schema, String[] stringValues)
+ throws Exception {
+ SegmentGeneratorConfig config = new SegmentGeneratorConfig(tableConfig,
schema);
+ config.setOutDir(SEGMENT_DIR_NAME);
+ config.setSegmentName(SEGMENT_NAME);
+
+ List<GenericRow> rows = new ArrayList<>(3);
+ for (int i = 0; i < 5; i++) {
+ GenericRow row = new GenericRow();
+ row.putValue(STRING_COLUMN, stringValues[i]);
+ row.putValue(LONG_COLUMN, LONG_VALUES[i]);
+ rows.add(row);
+ }
+
+ SegmentIndexCreationDriverImpl driver = new
SegmentIndexCreationDriverImpl();
+ driver.init(config, new GenericRowRecordReader(rows));
+ driver.build();
+ driver.getOutputDirectory().deleteOnExit();
+ return driver.getOutputDirectory();
+ }
+}
diff --git
a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/loader/SegmentPreProcessorTest.java
b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/loader/SegmentPreProcessorTest.java
index 1350fab3b5..832f1c7466 100644
---
a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/loader/SegmentPreProcessorTest.java
+++
b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/loader/SegmentPreProcessorTest.java
@@ -23,11 +23,13 @@ import java.io.File;
import java.net.URL;
import java.nio.file.Files;
import java.nio.file.attribute.FileTime;
+import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
+import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.function.Consumer;
@@ -35,8 +37,10 @@ import
org.apache.commons.configuration.PropertiesConfiguration;
import org.apache.commons.io.FileUtils;
import org.apache.pinot.segment.local.segment.creator.SegmentTestUtils;
import
org.apache.pinot.segment.local.segment.creator.impl.SegmentCreationDriverFactory;
+import
org.apache.pinot.segment.local.segment.creator.impl.SegmentIndexCreationDriverImpl;
import
org.apache.pinot.segment.local.segment.index.converter.SegmentV1V2ToV3FormatConverter;
import
org.apache.pinot.segment.local.segment.index.loader.columnminmaxvalue.ColumnMinMaxValueGeneratorMode;
+import org.apache.pinot.segment.local.segment.readers.GenericRowRecordReader;
import org.apache.pinot.segment.spi.ColumnMetadata;
import org.apache.pinot.segment.spi.V1Constants;
import org.apache.pinot.segment.spi.creator.SegmentGeneratorConfig;
@@ -59,6 +63,7 @@ import org.apache.pinot.spi.data.DimensionFieldSpec;
import org.apache.pinot.spi.data.FieldSpec;
import org.apache.pinot.spi.data.FieldSpec.DataType;
import org.apache.pinot.spi.data.Schema;
+import org.apache.pinot.spi.data.readers.GenericRow;
import org.apache.pinot.spi.env.PinotConfiguration;
import org.apache.pinot.spi.utils.ByteArray;
import org.apache.pinot.spi.utils.ReadMode;
@@ -1223,6 +1228,75 @@ public class SegmentPreProcessorTest {
});
}
+ @Test
+ public void testNeedAddMinMaxValue()
+ throws Exception {
+
+ String[] stringValuesInvalid = {"A,", "B,", "C,", "D,", "E"};
+ String[] stringValuesValid = {"A", "B", "C", "D", "E"};
+ long[] longValues = {1588316400000L, 1588489200000L, 1588662000000L,
1588834800000L, 1589007600000L};
+ TableConfig tableConfig = new
TableConfigBuilder(TableType.OFFLINE).setTableName("testTable").build();
+ Schema schema = new
Schema.SchemaBuilder().addSingleValueDimension("stringCol",
FieldSpec.DataType.STRING)
+ .addMetric("longCol", FieldSpec.DataType.LONG).build();
+
+ FileUtils.deleteQuietly(INDEX_DIR);
+
+ // build good segment, no needPreprocess
+ File segment = buildTestSegmentForMinMax(tableConfig, schema,
"validSegment", stringValuesValid, longValues);
+ SegmentDirectory segmentDirectory =
SegmentDirectoryLoaderRegistry.getDefaultSegmentDirectoryLoader()
+ .load(segment.toURI(),
+ new
SegmentDirectoryLoaderContext.Builder().setSegmentDirectoryConfigs(_configuration).build());
+ IndexLoadingConfig indexLoadingConfig = new IndexLoadingConfig();
+
indexLoadingConfig.setColumnMinMaxValueGeneratorMode(ColumnMinMaxValueGeneratorMode.ALL);
+ SegmentPreProcessor processor = new SegmentPreProcessor(segmentDirectory,
indexLoadingConfig, schema);
+ assertFalse(processor.needProcess());
+
+ // build bad segment, still no needPreprocess, since minMaxInvalid flag
should be set
+ FileUtils.deleteQuietly(INDEX_DIR);
+ segment = buildTestSegmentForMinMax(tableConfig, schema, "invalidSegment",
stringValuesInvalid, longValues);
+ segmentDirectory =
SegmentDirectoryLoaderRegistry.getDefaultSegmentDirectoryLoader().load(segment.toURI(),
+ new
SegmentDirectoryLoaderContext.Builder().setSegmentDirectoryConfigs(_configuration).build());
+ indexLoadingConfig = new IndexLoadingConfig();
+
indexLoadingConfig.setColumnMinMaxValueGeneratorMode(ColumnMinMaxValueGeneratorMode.NONE);
+ processor = new SegmentPreProcessor(segmentDirectory, indexLoadingConfig,
schema);
+ assertFalse(processor.needProcess());
+
+
indexLoadingConfig.setColumnMinMaxValueGeneratorMode(ColumnMinMaxValueGeneratorMode.ALL);
+ processor = new SegmentPreProcessor(segmentDirectory, indexLoadingConfig,
schema);
+ assertFalse(processor.needProcess());
+
+ // modify metadata, to remove min/max, now needPreprocess
+ removeMinMaxValuesFromMetadataFile(segment);
+ segmentDirectory =
SegmentDirectoryLoaderRegistry.getDefaultSegmentDirectoryLoader().load(segment.toURI(),
+ new
SegmentDirectoryLoaderContext.Builder().setSegmentDirectoryConfigs(_configuration).build());
+ processor = new SegmentPreProcessor(segmentDirectory, indexLoadingConfig,
schema);
+ assertTrue(processor.needProcess());
+
+ FileUtils.deleteQuietly(INDEX_DIR);
+ }
+
+ private File buildTestSegmentForMinMax(final TableConfig tableConfig, final
Schema schema, String segmentName,
+ String[] stringValues, long[] longValues)
+ throws Exception {
+ SegmentGeneratorConfig config = new SegmentGeneratorConfig(tableConfig,
schema);
+ config.setOutDir(INDEX_DIR.getAbsolutePath());
+ config.setSegmentName(segmentName);
+
+ List<GenericRow> rows = new ArrayList<>(3);
+ for (int i = 0; i < 5; i++) {
+ GenericRow row = new GenericRow();
+ row.putValue("stringCol", stringValues[i]);
+ row.putValue("longCol", longValues[i]);
+ rows.add(row);
+ }
+
+ SegmentIndexCreationDriverImpl driver = new
SegmentIndexCreationDriverImpl();
+ driver.init(config, new GenericRowRecordReader(rows));
+ driver.build();
+ driver.getOutputDirectory().deleteOnExit();
+ return driver.getOutputDirectory();
+ }
+
private static void removeMinMaxValuesFromMetadataFile(File indexDir)
throws Exception {
PropertiesConfiguration configuration =
SegmentMetadataImpl.getPropertiesConfiguration(indexDir);
@@ -1230,7 +1304,8 @@ public class SegmentPreProcessorTest {
while (keys.hasNext()) {
String key = keys.next();
if (key.endsWith(V1Constants.MetadataKeys.Column.MIN_VALUE) ||
key.endsWith(
- V1Constants.MetadataKeys.Column.MAX_VALUE)) {
+ V1Constants.MetadataKeys.Column.MAX_VALUE) || key.endsWith(
+ V1Constants.MetadataKeys.Column.MIN_MAX_VALUE_INVALID)) {
configuration.clearProperty(key);
}
}
diff --git
a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/ColumnMetadata.java
b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/ColumnMetadata.java
index 62796eecc4..00975a3c43 100644
---
a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/ColumnMetadata.java
+++
b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/ColumnMetadata.java
@@ -67,6 +67,10 @@ public interface ColumnMetadata {
Comparable getMaxValue();
+ default boolean isMinMaxValueInvalid() {
+ return false;
+ }
+
@JsonProperty
boolean hasDictionary();
diff --git
a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/V1Constants.java
b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/V1Constants.java
index 02049438ab..3a99c99346 100644
---
a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/V1Constants.java
+++
b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/V1Constants.java
@@ -100,6 +100,7 @@ public class V1Constants {
public static final String DEFAULT_NULL_VALUE = "defaultNullValue";
public static final String MIN_VALUE = "minValue";
public static final String MAX_VALUE = "maxValue";
+ public static final String MIN_MAX_VALUE_INVALID = "minMaxValueInvalid";
public static final String PARTITION_FUNCTION = "partitionFunction";
public static final String PARTITION_FUNCTION_CONFIG =
"partitionFunctionConfig";
public static final String NUM_PARTITIONS = "numPartitions";
diff --git
a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/metadata/ColumnMetadataImpl.java
b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/metadata/ColumnMetadataImpl.java
index e68e60013f..55b076bfbb 100644
---
a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/metadata/ColumnMetadataImpl.java
+++
b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/metadata/ColumnMetadataImpl.java
@@ -55,6 +55,8 @@ public class ColumnMetadataImpl implements ColumnMetadata {
private final boolean _sorted;
private final Comparable<?> _minValue;
private final Comparable<?> _maxValue;
+
+ private final boolean _minMaxValueInvalid;
private final boolean _hasDictionary;
private final int _columnMaxLength;
private final char _paddingCharacter;
@@ -66,15 +68,17 @@ public class ColumnMetadataImpl implements ColumnMetadata {
private final boolean _autoGenerated;
private ColumnMetadataImpl(FieldSpec fieldSpec, int totalDocs, int
cardinality, boolean sorted,
- Comparable<?> minValue, Comparable<?> maxValue, boolean hasDictionary,
int columnMaxLength, char paddingCharacter,
- int bitsPerElement, int maxNumberOfMultiValues, int totalNumberOfEntries,
- @Nullable PartitionFunction partitionFunction, @Nullable Set<Integer>
partitions, boolean autoGenerated) {
+ Comparable<?> minValue, Comparable<?> maxValue, boolean
minMaxValueInvalid, boolean hasDictionary,
+ int columnMaxLength, char paddingCharacter, int bitsPerElement, int
maxNumberOfMultiValues,
+ int totalNumberOfEntries, @Nullable PartitionFunction partitionFunction,
@Nullable Set<Integer> partitions,
+ boolean autoGenerated) {
_fieldSpec = fieldSpec;
_totalDocs = totalDocs;
_cardinality = cardinality;
_sorted = sorted;
_minValue = minValue;
_maxValue = maxValue;
+ _minMaxValueInvalid = minMaxValueInvalid;
_hasDictionary = hasDictionary;
_columnMaxLength = columnMaxLength;
_paddingCharacter = paddingCharacter;
@@ -116,6 +120,10 @@ public class ColumnMetadataImpl implements ColumnMetadata {
return _maxValue;
}
+ public boolean isMinMaxValueInvalid() {
+ return _minMaxValueInvalid;
+ }
+
@Override
public boolean hasDictionary() {
return _hasDictionary;
@@ -278,6 +286,7 @@ public class ColumnMetadataImpl implements ColumnMetadata {
throw new IllegalStateException("Unsupported data type: " + dataType
+ " for column: " + column);
}
}
+ builder.setMinMaxValueInvalid(config.getBoolean(Column.getKeyFor(column,
Column.MIN_MAX_VALUE_INVALID), false));
char paddingCharacter = V1Constants.Str.LEGACY_STRING_PAD_CHAR;
String padding = config.getString(Segment.SEGMENT_PADDING_CHARACTER, null);
@@ -329,6 +338,7 @@ public class ColumnMetadataImpl implements ColumnMetadata {
private boolean _sorted;
private Comparable<?> _minValue;
private Comparable<?> _maxValue;
+ private boolean _minMaxValueInvalid;
private boolean _hasDictionary;
private int _columnMaxLength;
private char _paddingCharacter;
@@ -369,6 +379,11 @@ public class ColumnMetadataImpl implements ColumnMetadata {
return this;
}
+ public Builder setMinMaxValueInvalid(boolean minMaxValueInvalid) {
+ _minMaxValueInvalid = minMaxValueInvalid;
+ return this;
+ }
+
public Builder setHasDictionary(boolean hasDictionary) {
_hasDictionary = hasDictionary;
return this;
@@ -415,9 +430,9 @@ public class ColumnMetadataImpl implements ColumnMetadata {
}
public ColumnMetadataImpl build() {
- return new ColumnMetadataImpl(_fieldSpec, _totalDocs, _cardinality,
_sorted, _minValue, _maxValue, _hasDictionary,
- _columnMaxLength, _paddingCharacter, _bitsPerElement,
_maxNumberOfMultiValues, _totalNumberOfEntries,
- _partitionFunction, _partitions, _autoGenerated);
+ return new ColumnMetadataImpl(_fieldSpec, _totalDocs, _cardinality,
_sorted, _minValue, _maxValue,
+ _minMaxValueInvalid, _hasDictionary, _columnMaxLength,
_paddingCharacter, _bitsPerElement,
+ _maxNumberOfMultiValues, _totalNumberOfEntries, _partitionFunction,
_partitions, _autoGenerated);
}
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
