This is an automated email from the ASF dual-hosted git repository.
somandal pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 3631b7d8e06 Handle null Geometry gracefully In H3Index (#16002)
3631b7d8e06 is described below
commit 3631b7d8e064557e38700a6d200828fcf34ef785
Author: Kartik Khare <[email protected]>
AuthorDate: Thu Jul 3 21:32:42 2025 +0530
Handle null Geometry gracefully In H3Index (#16002)
* Handle invalid geometry during H3 index creation
* Track null geometry rows in H3 index
* Add test for skipping invalid geometry
* Handle null Geometry gracefully in H3Index
* Address review comments
* Update table level metric only if tableNameWithType is not null
* Add IndexCreationContext::Common constructor without tableNameWithType
and mark it deprecated
* Address review comments
---------
Co-authored-by: KKCorps <[email protected]>
Co-authored-by: Sonam Mandal <[email protected]>
---
.../creator/impl/SegmentColumnarIndexCreator.java | 1 +
.../impl/inv/geospatial/BaseH3IndexCreator.java | 21 ++++--
.../impl/inv/geospatial/OffHeapH3IndexCreator.java | 7 +-
.../impl/inv/geospatial/OnHeapH3IndexCreator.java | 4 +-
.../local/segment/index/h3/H3IndexType.java | 6 +-
.../segment/index/loader/BaseIndexHandler.java | 2 +-
.../segment/index/loader/ForwardIndexHandler.java | 9 ++-
...IndexAndDictionaryBasedForwardIndexCreator.java | 6 +-
.../loader/bloomfilter/BloomFilterHandler.java | 2 +
.../defaultcolumn/BaseDefaultColumnHandler.java | 1 +
.../loader/invertedindex/FSTIndexHandler.java | 1 +
.../index/loader/invertedindex/H3IndexHandler.java | 2 +
.../loader/invertedindex/InvertedIndexHandler.java | 1 +
.../loader/invertedindex/JsonIndexHandler.java | 2 +
.../loader/invertedindex/RangeIndexHandler.java | 1 +
.../loader/invertedindex/TextIndexHandler.java | 1 +
.../loader/invertedindex/VectorIndexHandler.java | 2 +
.../segment/local/segment/index/H3IndexTest.java | 86 +++++++++++++++++++++-
.../segment/spi/creator/IndexCreationContext.java | 21 +++++-
.../spi/index/creator/GeoSpatialIndexCreator.java | 11 ++-
20 files changed, 163 insertions(+), 24 deletions(-)
diff --git
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/SegmentColumnarIndexCreator.java
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/SegmentColumnarIndexCreator.java
index 295138f41cd..e138fea3ea2 100644
---
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/SegmentColumnarIndexCreator.java
+++
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/SegmentColumnarIndexCreator.java
@@ -167,6 +167,7 @@ public class SegmentColumnarIndexCreator implements
SegmentCreator {
.withImmutableToMutableIdMap(immutableToMutableIdMap)
.withRealtimeConversion(segmentCreationSpec.isRealtimeConversion())
.withConsumerDir(segmentCreationSpec.getConsumerDir())
+ .withTableNameWithType(_config.getTableConfig().getTableName())
.build();
//@formatter:on
diff --git
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/inv/geospatial/BaseH3IndexCreator.java
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/inv/geospatial/BaseH3IndexCreator.java
index 91cbb62bdae..b3861073619 100644
---
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/inv/geospatial/BaseH3IndexCreator.java
+++
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/inv/geospatial/BaseH3IndexCreator.java
@@ -18,7 +18,6 @@
*/
package org.apache.pinot.segment.local.segment.creator.impl.inv.geospatial;
-import com.google.common.base.Preconditions;
import java.io.BufferedOutputStream;
import java.io.DataOutputStream;
import java.io.File;
@@ -27,9 +26,14 @@ import java.io.IOException;
import java.io.RandomAccessFile;
import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
+import java.util.Locale;
import java.util.Map;
import java.util.TreeMap;
+import javax.annotation.Nullable;
import org.apache.commons.io.FileUtils;
+import org.apache.pinot.common.metrics.ServerMeter;
+import org.apache.pinot.common.metrics.ServerMetrics;
+import org.apache.pinot.segment.local.segment.index.h3.H3IndexType;
import org.apache.pinot.segment.local.utils.GeometrySerializer;
import org.apache.pinot.segment.local.utils.H3Utils;
import org.apache.pinot.segment.spi.V1Constants;
@@ -67,6 +71,7 @@ public abstract class BaseH3IndexCreator implements
GeoSpatialIndexCreator {
static final String BITMAP_OFFSET_FILE_NAME = "bitmap.offset.buf";
static final String BITMAP_VALUE_FILE_NAME = "bitmap.value.buf";
+ final String _tableNameWithType;
final File _indexFile;
final File _tempDir;
final File _dictionaryFile;
@@ -83,8 +88,9 @@ public abstract class BaseH3IndexCreator implements
GeoSpatialIndexCreator {
int _nextDocId;
- BaseH3IndexCreator(File indexDir, String columnName, H3IndexResolution
resolution)
+ BaseH3IndexCreator(File indexDir, String columnName, String
tableNameWithType, H3IndexResolution resolution)
throws IOException {
+ _tableNameWithType = tableNameWithType;
_indexFile = new File(indexDir, columnName +
V1Constants.Indexes.H3_INDEX_FILE_EXTENSION);
_tempDir = new File(indexDir, columnName + TEMP_DIR_SUFFIX);
if (_tempDir.exists()) {
@@ -108,10 +114,15 @@ public abstract class BaseH3IndexCreator implements
GeoSpatialIndexCreator {
}
@Override
- public void add(Geometry geometry)
+ public void add(@Nullable Geometry geometry)
throws IOException {
- Preconditions.checkState(geometry instanceof Point, "H3 index can only be
applied to Point, got: %s",
- geometry.getGeometryType());
+ if (geometry == null || !(geometry instanceof Point)) {
+ String metricKeyName =
+ _tableNameWithType + "-" +
H3IndexType.INDEX_DISPLAY_NAME.toUpperCase(Locale.US) + "-indexingError";
+ ServerMetrics.get().addMeteredTableValue(metricKeyName,
ServerMeter.INDEXING_FAILURES, 1);
+ _nextDocId++;
+ return;
+ }
Coordinate coordinate = geometry.getCoordinate();
// TODO: support multiple resolutions
long h3Id = H3Utils.H3_CORE.latLngToCell(coordinate.y, coordinate.x,
_lowestResolution);
diff --git
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/inv/geospatial/OffHeapH3IndexCreator.java
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/inv/geospatial/OffHeapH3IndexCreator.java
index 4e9334da708..35240582ca0 100644
---
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/inv/geospatial/OffHeapH3IndexCreator.java
+++
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/inv/geospatial/OffHeapH3IndexCreator.java
@@ -31,6 +31,7 @@ import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.PriorityQueue;
+import javax.annotation.Nullable;
import org.apache.pinot.segment.spi.index.reader.H3IndexResolution;
import org.apache.pinot.segment.spi.memory.PinotDataBuffer;
import org.locationtech.jts.geom.Geometry;
@@ -60,15 +61,15 @@ public class OffHeapH3IndexCreator extends
BaseH3IndexCreator {
private long _postingListChunkOffset;
- public OffHeapH3IndexCreator(File indexDir, String columnName,
H3IndexResolution resolution)
+ public OffHeapH3IndexCreator(File indexDir, String columnName, String
tableNameWithType, H3IndexResolution resolution)
throws IOException {
- super(indexDir, columnName, resolution);
+ super(indexDir, columnName, tableNameWithType, resolution);
_postingListFile = new File(_tempDir, POSTING_LIST_FILE_NAME);
_postingListOutputStream = new DataOutputStream(new
BufferedOutputStream(new FileOutputStream(_postingListFile)));
}
@Override
- public void add(Geometry geometry)
+ public void add(@Nullable Geometry geometry)
throws IOException {
super.add(geometry);
if (_postingListMap.size() % FLUSH_THRESHOLD == 0) {
diff --git
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/inv/geospatial/OnHeapH3IndexCreator.java
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/inv/geospatial/OnHeapH3IndexCreator.java
index d05d23784f0..c9583a45d75 100644
---
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/inv/geospatial/OnHeapH3IndexCreator.java
+++
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/inv/geospatial/OnHeapH3IndexCreator.java
@@ -34,9 +34,9 @@ import org.roaringbitmap.RoaringBitmapWriter;
*/
public class OnHeapH3IndexCreator extends BaseH3IndexCreator {
- public OnHeapH3IndexCreator(File indexDir, String columnName,
H3IndexResolution resolution)
+ public OnHeapH3IndexCreator(File indexDir, String columnName, String
tableNameWithType, H3IndexResolution resolution)
throws IOException {
- super(indexDir, columnName, resolution);
+ super(indexDir, columnName, tableNameWithType, resolution);
}
@Override
diff --git
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/h3/H3IndexType.java
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/h3/H3IndexType.java
index 01cfa41f438..5d0996ef746 100644
---
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/h3/H3IndexType.java
+++
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/h3/H3IndexType.java
@@ -108,8 +108,10 @@ public class H3IndexType extends
AbstractIndexType<H3IndexConfig, H3IndexReader,
"H3 index is currently only supported on BYTES columns");
H3IndexResolution resolution =
Objects.requireNonNull(indexConfig).getResolution();
return context.isOnHeap()
- ? new OnHeapH3IndexCreator(context.getIndexDir(),
context.getFieldSpec().getName(), resolution)
- : new OffHeapH3IndexCreator(context.getIndexDir(),
context.getFieldSpec().getName(), resolution);
+ ? new OnHeapH3IndexCreator(context.getIndexDir(),
context.getFieldSpec().getName(),
+ context.getTableNameWithType(), resolution)
+ : new OffHeapH3IndexCreator(context.getIndexDir(),
context.getFieldSpec().getName(),
+ context.getTableNameWithType(), resolution);
}
@Override
diff --git
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/BaseIndexHandler.java
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/BaseIndexHandler.java
index 00c47586cdd..5b5f870553d 100644
---
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/BaseIndexHandler.java
+++
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/BaseIndexHandler.java
@@ -115,7 +115,7 @@ public abstract class BaseIndexHandler implements
IndexHandler {
InvertedIndexAndDictionaryBasedForwardIndexCreator creator =
new InvertedIndexAndDictionaryBasedForwardIndexCreator(columnName,
_segmentDirectory, dictionaryEnabled,
- forwardIndexConfig, segmentWriter, isTemporaryForwardIndex);
+ forwardIndexConfig, segmentWriter, isTemporaryForwardIndex,
_tableConfig.getTableName());
creator.regenerateForwardIndex();
// Validate that the forward index is created.
if (!segmentWriter.hasIndexFor(columnName, StandardIndexes.forward())) {
diff --git
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/ForwardIndexHandler.java
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/ForwardIndexHandler.java
index ce347bdc31a..09785675297 100644
---
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/ForwardIndexHandler.java
+++
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/ForwardIndexHandler.java
@@ -462,7 +462,8 @@ public class ForwardIndexHandler extends BaseIndexHandler {
throws Exception {
try (ForwardIndexReader<?> reader = ForwardIndexType.read(segmentWriter,
columnMetadata)) {
IndexCreationContext.Builder builder =
-
IndexCreationContext.builder().withIndexDir(indexDir).withColumnMetadata(columnMetadata);
+
IndexCreationContext.builder().withIndexDir(indexDir).withColumnMetadata(columnMetadata)
+ .withTableNameWithType(_tableConfig.getTableName());
// Set entry length info for raw index creators. No need to set this
when changing dictionary id compression type.
if (!reader.isDictionaryEncoded() &&
!columnMetadata.getDataType().getStoredType().isFixedWidth()) {
int lengthOfLongestEntry = reader.getLengthOfLongestEntry();
@@ -887,7 +888,8 @@ public class ForwardIndexHandler extends BaseIndexHandler {
throws Exception {
try (ForwardIndexReader<?> reader = ForwardIndexType.read(segmentWriter,
existingColMetadata)) {
IndexCreationContext.Builder builder =
-
IndexCreationContext.builder().withIndexDir(indexDir).withColumnMetadata(existingColMetadata);
+
IndexCreationContext.builder().withIndexDir(indexDir).withColumnMetadata(existingColMetadata)
+ .withTableNameWithType(_tableConfig.getTableName());
// existingColMetadata has dictEnable=false. Overwrite the value.
builder.withDictionary(true);
IndexCreationContext context = builder.build();
@@ -961,7 +963,8 @@ public class ForwardIndexHandler extends BaseIndexHandler {
try (ForwardIndexReader<?> reader = ForwardIndexType.read(segmentWriter,
columnMetadata)) {
Dictionary dictionary = DictionaryIndexType.read(segmentWriter,
columnMetadata);
IndexCreationContext.Builder builder =
-
IndexCreationContext.builder().withIndexDir(indexDir).withColumnMetadata(columnMetadata);
+
IndexCreationContext.builder().withIndexDir(indexDir).withColumnMetadata(columnMetadata)
+ .withTableNameWithType(_tableConfig.getTableName());
builder.withDictionary(false);
if (!columnMetadata.getDataType().getStoredType().isFixedWidth()) {
if (columnMetadata.isSingleValue()) {
diff --git
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/InvertedIndexAndDictionaryBasedForwardIndexCreator.java
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/InvertedIndexAndDictionaryBasedForwardIndexCreator.java
index 2d71deaab62..a58b4ccce5e 100644
---
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/InvertedIndexAndDictionaryBasedForwardIndexCreator.java
+++
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/InvertedIndexAndDictionaryBasedForwardIndexCreator.java
@@ -82,6 +82,7 @@ public class
InvertedIndexAndDictionaryBasedForwardIndexCreator implements AutoC
private final ForwardIndexConfig _forwardIndexConfig;
private final SegmentDirectory.Writer _segmentWriter;
private final boolean _isTemporaryForwardIndex;
+ private final String _tableNameWithType;
// Metadata
private final SegmentDirectory _segmentDirectory;
@@ -114,13 +115,14 @@ public class
InvertedIndexAndDictionaryBasedForwardIndexCreator implements AutoC
public InvertedIndexAndDictionaryBasedForwardIndexCreator(String columnName,
SegmentDirectory segmentDirectory,
boolean dictionaryEnabled, ForwardIndexConfig fwdConf,
SegmentDirectory.Writer segmentWriter,
- boolean isTemporaryForwardIndex)
+ boolean isTemporaryForwardIndex, String tableNameWithType)
throws IOException {
_columnName = columnName;
_segmentDirectory = segmentDirectory;
_segmentMetadata = segmentDirectory.getSegmentMetadata();
_segmentWriter = segmentWriter;
_isTemporaryForwardIndex = isTemporaryForwardIndex;
+ _tableNameWithType = tableNameWithType;
_columnMetadata = _segmentMetadata.getColumnMetadataFor(columnName);
_singleValue = _columnMetadata.isSingleValue();
@@ -268,6 +270,7 @@ public class
InvertedIndexAndDictionaryBasedForwardIndexCreator implements AutoC
.withForwardIndexDisabled(false)
.withDictionary(_dictionaryEnabled)
.withLengthOfLongestEntry(lengthOfLongestEntry)
+ .withTableNameWithType(_tableNameWithType)
.build();
// note: this method closes buffers and removes files
@@ -354,6 +357,7 @@ public class
InvertedIndexAndDictionaryBasedForwardIndexCreator implements AutoC
.withMaxNumberOfMultiValueElements(maxNumberOfMultiValues[0])
.withMaxRowLengthInBytes(maxRowLengthInBytes[0])
.withLengthOfLongestEntry(lengthOfLongestEntry)
+ .withTableNameWithType(_tableNameWithType)
.build();
writeToForwardIndex(dictionary, context);
diff --git
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/bloomfilter/BloomFilterHandler.java
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/bloomfilter/BloomFilterHandler.java
index c1b29b4563d..9a6bd92fa6e 100644
---
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/bloomfilter/BloomFilterHandler.java
+++
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/bloomfilter/BloomFilterHandler.java
@@ -117,6 +117,7 @@ public class BloomFilterHandler extends BaseIndexHandler {
IndexCreationContext context = IndexCreationContext.builder()
.withIndexDir(indexDir)
.withColumnMetadata(columnMetadata)
+ .withTableNameWithType(_tableConfig.getTableName())
.build();
try (BloomFilterCreator bloomFilterCreator =
StandardIndexes.bloomFilter().createIndexCreator(context,
bloomFilterConfig);
@@ -136,6 +137,7 @@ public class BloomFilterHandler extends BaseIndexHandler {
IndexCreationContext context = IndexCreationContext.builder()
.withIndexDir(indexDir)
.withColumnMetadata(columnMetadata)
+ .withTableNameWithType(_tableConfig.getTableName())
.build();
try (BloomFilterCreator bloomFilterCreator = StandardIndexes.bloomFilter()
.createIndexCreator(context, bloomFilterConfig);
diff --git
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/defaultcolumn/BaseDefaultColumnHandler.java
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/defaultcolumn/BaseDefaultColumnHandler.java
index 84129e7e0ce..3a575c439ac 100644
---
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/defaultcolumn/BaseDefaultColumnHandler.java
+++
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/defaultcolumn/BaseDefaultColumnHandler.java
@@ -1199,6 +1199,7 @@ public abstract class BaseDefaultColumnHandler implements
DefaultColumnHandler {
.withColumnIndexCreationInfo(indexCreationInfo)
.withTotalDocs(numDocs)
.withDictionary(hasDictionary)
+ .withTableNameWithType(_tableConfig.getTableName())
.build();
ForwardIndexConfig forwardIndexConfig = null;
diff --git
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/invertedindex/FSTIndexHandler.java
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/invertedindex/FSTIndexHandler.java
index 1ebf84e562e..0ee9ce08677 100644
---
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/invertedindex/FSTIndexHandler.java
+++
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/invertedindex/FSTIndexHandler.java
@@ -172,6 +172,7 @@ public class FSTIndexHandler extends BaseIndexHandler {
IndexCreationContext context = IndexCreationContext.builder()
.withIndexDir(indexDir)
.withColumnMetadata(columnMetadata)
+ .withTableNameWithType(_tableConfig.getTableName())
.build();
FstIndexConfig config =
_fieldIndexConfigs.get(columnName).getConfig(StandardIndexes.fst());
diff --git
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/invertedindex/H3IndexHandler.java
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/invertedindex/H3IndexHandler.java
index 78ddd6b110a..b244184a293 100644
---
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/invertedindex/H3IndexHandler.java
+++
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/invertedindex/H3IndexHandler.java
@@ -162,6 +162,7 @@ public class H3IndexHandler extends BaseIndexHandler {
IndexCreationContext context = IndexCreationContext.builder()
.withIndexDir(indexDir)
.withColumnMetadata(columnMetadata)
+ .withTableNameWithType(_tableConfig.getTableName())
.build();
H3IndexConfig config = colIndexConf.getConfig(StandardIndexes.h3());
@@ -187,6 +188,7 @@ public class H3IndexHandler extends BaseIndexHandler {
IndexCreationContext context = IndexCreationContext.builder()
.withIndexDir(indexDir)
.withColumnMetadata(columnMetadata)
+ .withTableNameWithType(_tableConfig.getTableName())
.build();
H3IndexConfig config =
_fieldIndexConfigs.get(columnName).getConfig(StandardIndexes.h3());
try (ForwardIndexReader forwardIndexReader =
ForwardIndexType.read(segmentWriter, columnMetadata);
diff --git
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/invertedindex/InvertedIndexHandler.java
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/invertedindex/InvertedIndexHandler.java
index 56b4752415d..0feeab3ea13 100644
---
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/invertedindex/InvertedIndexHandler.java
+++
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/invertedindex/InvertedIndexHandler.java
@@ -139,6 +139,7 @@ public class InvertedIndexHandler extends BaseIndexHandler {
IndexCreationContext.Common context = IndexCreationContext.builder()
.withIndexDir(indexDir)
.withColumnMetadata(columnMetadata)
+ .withTableNameWithType(_tableConfig.getTableName())
.build();
try (DictionaryBasedInvertedIndexCreator creator =
StandardIndexes.inverted()
diff --git
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/invertedindex/JsonIndexHandler.java
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/invertedindex/JsonIndexHandler.java
index 7b2e1fb0709..5954486111c 100644
---
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/invertedindex/JsonIndexHandler.java
+++
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/invertedindex/JsonIndexHandler.java
@@ -160,6 +160,7 @@ public class JsonIndexHandler extends BaseIndexHandler {
IndexCreationContext context = IndexCreationContext.builder()
.withIndexDir(indexDir)
.withColumnMetadata(columnMetadata)
+ .withTableNameWithType(_tableConfig.getTableName())
.build();
JsonIndexConfig config = _jsonIndexConfigs.get(columnName);
try (ForwardIndexReader forwardIndexReader =
ForwardIndexType.read(segmentWriter, columnMetadata);
@@ -182,6 +183,7 @@ public class JsonIndexHandler extends BaseIndexHandler {
IndexCreationContext context = IndexCreationContext.builder()
.withIndexDir(indexDir)
.withColumnMetadata(columnMetadata)
+ .withTableNameWithType(_tableConfig.getTableName())
.build();
JsonIndexConfig config = _jsonIndexConfigs.get(columnName);
try (ForwardIndexReader forwardIndexReader =
ForwardIndexType.read(segmentWriter, columnMetadata);
diff --git
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/invertedindex/RangeIndexHandler.java
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/invertedindex/RangeIndexHandler.java
index c1e33427643..532b739ed9e 100644
---
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/invertedindex/RangeIndexHandler.java
+++
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/invertedindex/RangeIndexHandler.java
@@ -260,6 +260,7 @@ public class RangeIndexHandler extends BaseIndexHandler {
IndexCreationContext context = IndexCreationContext.builder()
.withIndexDir(indexDir)
.withColumnMetadata(columnMetadata)
+ .withTableNameWithType(_tableConfig.getTableName())
.build();
RangeIndexConfig config =
_fieldIndexConfigs.get(columnMetadata.getColumnName())
.getConfig(StandardIndexes.range());
diff --git
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/invertedindex/TextIndexHandler.java
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/invertedindex/TextIndexHandler.java
index d3f7f311d61..9655252dc9d 100644
---
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/invertedindex/TextIndexHandler.java
+++
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/invertedindex/TextIndexHandler.java
@@ -174,6 +174,7 @@ public class TextIndexHandler extends BaseIndexHandler {
.withColumnMetadata(columnMetadata)
.withIndexDir(segmentDirectory)
.withTextCommitOnClose(true)
+ .withTableNameWithType(_tableConfig.getTableName())
.build();
TextIndexConfig config =
_fieldIndexConfigs.get(columnName).getConfig(StandardIndexes.text());
diff --git
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/invertedindex/VectorIndexHandler.java
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/invertedindex/VectorIndexHandler.java
index 72d74763ca1..f09469e0c4a 100644
---
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/invertedindex/VectorIndexHandler.java
+++
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/invertedindex/VectorIndexHandler.java
@@ -167,6 +167,7 @@ public class VectorIndexHandler extends BaseIndexHandler {
IndexCreationContext context = IndexCreationContext.builder()
.withIndexDir(segmentDirectory)
.withColumnMetadata(columnMetadata)
+ .withTableNameWithType(_tableConfig.getTableName())
.build();
VectorIndexConfig config =
colIndexConf.getConfig(StandardIndexes.vector());
@@ -200,6 +201,7 @@ public class VectorIndexHandler extends BaseIndexHandler {
IndexCreationContext context = IndexCreationContext.builder()
.withIndexDir(segmentDirectory)
.withColumnMetadata(columnMetadata)
+ .withTableNameWithType(_tableConfig.getTableName())
.build();
VectorIndexConfig config =
_fieldIndexConfigs.get(columnName).getConfig(StandardIndexes.vector());
try (ForwardIndexReader forwardIndexReader =
ForwardIndexType.read(segmentWriter, columnMetadata);
diff --git
a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/H3IndexTest.java
b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/H3IndexTest.java
index d7e027d4267..7493d43db6f 100644
---
a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/H3IndexTest.java
+++
b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/H3IndexTest.java
@@ -47,6 +47,7 @@ import
org.apache.pinot.segment.spi.index.reader.H3IndexResolution;
import org.apache.pinot.segment.spi.memory.PinotDataBuffer;
import org.apache.pinot.spi.config.table.FieldConfig;
import org.locationtech.jts.geom.Coordinate;
+import org.locationtech.jts.geom.MultiPoint;
import org.locationtech.jts.geom.Point;
import org.testng.Assert;
import org.testng.annotations.AfterClass;
@@ -91,9 +92,9 @@ public class H3IndexTest implements
PinotBuffersAfterMethodCheckRule {
try (MutableH3Index mutableH3Index = new
MutableH3Index(h3IndexResolution)) {
try (GeoSpatialIndexCreator onHeapCreator = new
OnHeapH3IndexCreator(TEMP_DIR, onHeapColumnName,
- h3IndexResolution);
+ "myTable_OFFLINE", h3IndexResolution);
GeoSpatialIndexCreator offHeapCreator = new
OffHeapH3IndexCreator(TEMP_DIR, offHeapColumnName,
- h3IndexResolution)) {
+ "myTable_OFFLINE", h3IndexResolution)) {
int docId = 0;
while (expectedCardinalities.size() < numUniqueH3Ids) {
double longitude = RANDOM.nextDouble() * 360 - 180;
@@ -126,6 +127,87 @@ public class H3IndexTest implements
PinotBuffersAfterMethodCheckRule {
}
}
+ @Test
+ public void testSkipInvalidGeometry()
+ throws Exception {
+ String columnName = "skipInvalid";
+ int res = 5;
+ H3IndexResolution resolution = new
H3IndexResolution(Collections.singletonList(res));
+
+ try (GeoSpatialIndexCreator creator = new OnHeapH3IndexCreator(TEMP_DIR,
columnName, "myTable_OFFLINE",
+ resolution)) {
+ Point point = GeometryUtils.GEOMETRY_FACTORY.createPoint(new
Coordinate(10, 20));
+ creator.add(point);
+
+ // Invalid serialized bytes should be skipped without throwing exception
+ creator.add(new byte[]{1, 2, 3}, -1);
+
+ creator.seal();
+ }
+
+ File indexFile = new File(TEMP_DIR, columnName +
V1Constants.Indexes.H3_INDEX_FILE_EXTENSION);
+ try (PinotDataBuffer buffer =
PinotDataBuffer.mapReadOnlyBigEndianFile(indexFile);
+ H3IndexReader reader = new ImmutableH3IndexReader(buffer)) {
+ long h3Id = H3Utils.H3_CORE.latLngToCell(20, 10, res);
+ Assert.assertEquals(reader.getDocIds(h3Id).getCardinality(), 1);
+ }
+ }
+
+ @Test
+ public void testSkipNullGeometry()
+ throws Exception {
+ String columnName = "skipNull";
+ int res = 5;
+ H3IndexResolution resolution = new
H3IndexResolution(Collections.singletonList(res));
+
+ try (GeoSpatialIndexCreator creator = new OnHeapH3IndexCreator(TEMP_DIR,
columnName, "myTable_OFFLINE",
+ resolution)) {
+ Point point = GeometryUtils.GEOMETRY_FACTORY.createPoint(new
Coordinate(10, 20));
+ creator.add(point);
+
+ // Explicit null geometry should also be skipped
+ creator.add(null);
+
+ creator.seal();
+ }
+
+ File indexFile = new File(TEMP_DIR, columnName +
V1Constants.Indexes.H3_INDEX_FILE_EXTENSION);
+ try (PinotDataBuffer buffer =
PinotDataBuffer.mapReadOnlyBigEndianFile(indexFile);
+ H3IndexReader reader = new ImmutableH3IndexReader(buffer)) {
+ long h3Id = H3Utils.H3_CORE.latLngToCell(20, 10, res);
+ Assert.assertEquals(reader.getDocIds(h3Id).getCardinality(), 1);
+ }
+ }
+
+ @Test
+ public void testSkipNonPointGeometry()
+ throws Exception {
+ String columnName = "skipInvalidGeometryType";
+ int res = 5;
+ H3IndexResolution resolution = new
H3IndexResolution(Collections.singletonList(res));
+
+ try (GeoSpatialIndexCreator creator = new OnHeapH3IndexCreator(TEMP_DIR,
columnName, "myTable_OFFLINE",
+ resolution)) {
+ Point point = GeometryUtils.GEOMETRY_FACTORY.createPoint(new
Coordinate(10, 42));
+ creator.add(point);
+
+ // Explicit non-point geometry should also be skipped
+ Point[] points = new Point[1];
+ points[0] = point;
+ MultiPoint multiPoint =
GeometryUtils.GEOMETRY_FACTORY.createMultiPoint(points);
+ creator.add(multiPoint);
+
+ creator.seal();
+ }
+
+ File indexFile = new File(TEMP_DIR, columnName +
V1Constants.Indexes.H3_INDEX_FILE_EXTENSION);
+ try (PinotDataBuffer buffer =
PinotDataBuffer.mapReadOnlyBigEndianFile(indexFile);
+ H3IndexReader reader = new ImmutableH3IndexReader(buffer)) {
+ long h3Id = H3Utils.H3_CORE.latLngToCell(42, 10, res);
+ Assert.assertEquals(reader.getDocIds(h3Id).getCardinality(), 1);
+ }
+ }
+
public static class ConfTest extends AbstractSerdeIndexContract {
protected void assertEquals(H3IndexConfig expected) {
diff --git
a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/creator/IndexCreationContext.java
b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/creator/IndexCreationContext.java
index 5d3a1a78ba7..a9c300721bf 100644
---
a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/creator/IndexCreationContext.java
+++
b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/creator/IndexCreationContext.java
@@ -111,6 +111,8 @@ public interface IndexCreationContext {
*/
int[] getImmutableToMutableIdMap();
+ String getTableNameWithType();
+
final class Builder {
private ColumnStatistics _columnStatistics;
private File _indexDir;
@@ -134,6 +136,7 @@ public interface IndexCreationContext {
private boolean _realtimeConversion = false;
private File _consumerDir;
private int[] _immutableToMutableIdMap;
+ private String _tableNameWithType;
public Builder withColumnIndexCreationInfo(ColumnIndexCreationInfo
columnIndexCreationInfo) {
return
withLengthOfLongestEntry(columnIndexCreationInfo.getLengthOfLongestEntry())
@@ -266,12 +269,17 @@ public interface IndexCreationContext {
return this;
}
+ public Builder withTableNameWithType(String tableNameWithType) {
+ _tableNameWithType = tableNameWithType;
+ return this;
+ }
+
public Common build() {
return new Common(Objects.requireNonNull(_indexDir),
_lengthOfLongestEntry, _maxNumberOfMultiValueElements,
_maxRowLengthInBytes, _onHeap, Objects.requireNonNull(_fieldSpec),
_sorted, _cardinality,
_totalNumberOfEntries, _totalDocs, _hasDictionary, _minValue,
_maxValue, _forwardIndexDisabled,
_sortedUniqueElementsArray, _optimizedDictionary, _fixedLength,
_textCommitOnClose, _columnStatistics,
- _realtimeConversion, _consumerDir, _immutableToMutableIdMap);
+ _realtimeConversion, _consumerDir, _immutableToMutableIdMap,
_tableNameWithType);
}
public Builder withSortedUniqueElementsArray(Object
sortedUniqueElementsArray) {
@@ -308,14 +316,15 @@ public interface IndexCreationContext {
private final boolean _realtimeConversion;
private final File _consumerDir;
private final int[] _immutableToMutableIdMap;
+ private final String _tableNameWithType;
- public Common(File indexDir, int lengthOfLongestEntry,
+ private Common(File indexDir, int lengthOfLongestEntry,
int maxNumberOfMultiValueElements, int maxRowLengthInBytes, boolean
onHeap,
FieldSpec fieldSpec, boolean sorted, int cardinality, int
totalNumberOfEntries,
int totalDocs, boolean hasDictionary, Comparable<?> minValue,
Comparable<?> maxValue,
boolean forwardIndexDisabled, Object sortedUniqueElementsArray,
boolean optimizeDictionary, boolean fixedLength,
boolean textCommitOnClose, ColumnStatistics columnStatistics, boolean
realtimeConversion, File consumerDir,
- int[] immutableToMutableIdMap) {
+ int[] immutableToMutableIdMap, String tableNameWithType) {
_indexDir = indexDir;
_lengthOfLongestEntry = lengthOfLongestEntry;
_maxNumberOfMultiValueElements = maxNumberOfMultiValueElements;
@@ -338,6 +347,7 @@ public interface IndexCreationContext {
_realtimeConversion = realtimeConversion;
_consumerDir = consumerDir;
_immutableToMutableIdMap = immutableToMutableIdMap;
+ _tableNameWithType = tableNameWithType;
}
public FieldSpec getFieldSpec() {
@@ -438,5 +448,10 @@ public interface IndexCreationContext {
public int[] getImmutableToMutableIdMap() {
return _immutableToMutableIdMap;
}
+
+ @Override
+ public String getTableNameWithType() {
+ return _tableNameWithType;
+ }
}
}
diff --git
a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/creator/GeoSpatialIndexCreator.java
b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/creator/GeoSpatialIndexCreator.java
index efec2027669..f7cf80f429d 100644
---
a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/creator/GeoSpatialIndexCreator.java
+++
b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/creator/GeoSpatialIndexCreator.java
@@ -34,7 +34,14 @@ public interface GeoSpatialIndexCreator extends IndexCreator
{
@Override
default void add(Object value, int dictId)
throws IOException {
- add(deserialize((byte[]) value));
+ Geometry geometry;
+ try {
+ geometry = deserialize((byte[]) value);
+ } catch (Exception e) {
+ // Swallow the exception and treat the geometry as null
+ geometry = null;
+ }
+ add(geometry);
}
@Override
@@ -45,7 +52,7 @@ public interface GeoSpatialIndexCreator extends IndexCreator {
/**
* Adds the next geospatial value.
*/
- void add(Geometry geometry)
+ void add(@Nullable Geometry geometry)
throws IOException;
/**
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
