This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new 9cd3a2ef60 [core] Support ignoring unknown file index types on read
(#10654)
9cd3a2ef60 is described below
commit 9cd3a2ef60fa8213791b4aed6ec6287fce62825e
Author: Zhang Jiawei <[email protected]>
AuthorDate: Tue Oct 20 14:49:18 2026 +0800
[core] Support ignoring unknown file index types on read (#10654)
---
docs/docs/concepts/spec/fileindex.mdx | 5 +
docs/generated/core_configuration.html | 6 +
.../main/java/org/apache/paimon/CoreOptions.java | 14 ++
.../apache/paimon/fileindex/FileIndexFormat.java | 26 ++++
.../paimon/fileindex/FileIndexPredicate.java | 14 +-
.../paimon/fileindex/FileIndexerFactoryUtils.java | 4 +
.../paimon/fileindex/UnknownFileIndexTest.java | 160 +++++++++++++++++++++
.../org/apache/paimon/AppendOnlyFileStore.java | 1 +
.../java/org/apache/paimon/KeyValueFileStore.java | 3 +-
.../org/apache/paimon/io/FileIndexEvaluator.java | 29 +++-
.../paimon/operation/AppendOnlyFileStoreScan.java | 5 +-
.../operation/DataEvolutionFileStoreScan.java | 1 +
.../paimon/operation/DataEvolutionSplitRead.java | 18 ++-
.../paimon/operation/KeyValueFileStoreScan.java | 7 +-
.../apache/paimon/operation/RawFileSplitRead.java | 6 +-
.../apache/paimon/io/FileIndexEvaluatorTest.java | 86 +++++++++++
.../apache/paimon/manifest/ManifestFileTest.java | 3 +
.../paimon/table/DataEvolutionFileIndexTest.java | 45 ++++++
18 files changed, 421 insertions(+), 12 deletions(-)
diff --git a/docs/docs/concepts/spec/fileindex.mdx
b/docs/docs/concepts/spec/fileindex.mdx
index 6fde4d722b..983eeca46f 100644
--- a/docs/docs/concepts/spec/fileindex.mdx
+++ b/docs/docs/concepts/spec/fileindex.mdx
@@ -43,6 +43,11 @@ deletion vectors, see [Table Index](./tableindex).
| Range bitmap | [Range Bitmap](#index-range-bitmap) |
| Bit-slice bitmap | [Bit-Slice Index Bitmap](#index-bit-slice-index-bitmap) |
+By default, reading an unregistered non-empty index type fails. Set
+`file-index.read.ignore-unknown-type` to `true` to skip unregistered index
types and log a warning.
+Skipped indexes do not filter any rows; registered indexes and query
predicates still apply.
+This option does not suppress errors from registered index implementations.
+
## Index File
The container version is selected by `file-index.format.version` (default:
`1`). Readers support
diff --git a/docs/generated/core_configuration.html
b/docs/generated/core_configuration.html
index 601ed2a358..7572026db7 100644
--- a/docs/generated/core_configuration.html
+++ b/docs/generated/core_configuration.html
@@ -740,6 +740,12 @@ under the License.
<td>Boolean</td>
<td>Whether enabled read file index.</td>
</tr>
+ <tr>
+ <td><h5>file-index.read.ignore-unknown-type</h5></td>
+ <td style="word-wrap: break-word;">false</td>
+ <td>Boolean</td>
+ <td>Whether to ignore unregistered file index types when reading.
If true, skip these indexes with a warning and retain all rows for them. Other
indexes can still filter rows. If false, fail when reading an unregistered
non-empty index.</td>
+ </tr>
<tr>
<td><h5>file-operation.thread-num</h5></td>
<td style="word-wrap: break-word;">(none)</td>
diff --git a/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java
b/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java
index 4e8a1cec46..25cf6876ad 100644
--- a/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java
+++ b/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java
@@ -443,6 +443,16 @@ public class CoreOptions implements Serializable {
.withDescription(
"File index container version to write (1 or 2).
Version 2 supports 64-bit payload positions and lengths.");
+ public static final ConfigOption<Boolean>
FILE_INDEX_READ_IGNORE_UNKNOWN_TYPE =
+ key("file-index.read.ignore-unknown-type")
+ .booleanType()
+ .defaultValue(false)
+ .withDescription(
+ "Whether to ignore unregistered file index types
when reading. "
+ + "If true, skip these indexes with a
warning and retain all rows "
+ + "for them. Other indexes can still
filter rows. "
+ + "If false, fail when reading an
unregistered non-empty index.");
+
public static final ConfigOption<Boolean> FILE_INDEX_READ_ENABLED =
key("file-index.read.enabled")
.booleanType()
@@ -4592,6 +4602,10 @@ public class CoreOptions implements Serializable {
return bytes;
}
+ public boolean fileIndexReadIgnoreUnknownType() {
+ return options.get(FILE_INDEX_READ_IGNORE_UNKNOWN_TYPE);
+ }
+
public boolean fileIndexReadEnabled() {
return options.get(FILE_INDEX_READ_ENABLED);
}
diff --git
a/paimon-common/src/main/java/org/apache/paimon/fileindex/FileIndexFormat.java
b/paimon-common/src/main/java/org/apache/paimon/fileindex/FileIndexFormat.java
index 0359d6073f..62423865d5 100644
---
a/paimon-common/src/main/java/org/apache/paimon/fileindex/FileIndexFormat.java
+++
b/paimon-common/src/main/java/org/apache/paimon/fileindex/FileIndexFormat.java
@@ -26,6 +26,9 @@ import org.apache.paimon.types.RowType;
import org.apache.paimon.utils.IOUtils;
import org.apache.paimon.utils.Pair;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
import javax.annotation.Nullable;
import java.io.Closeable;
@@ -206,6 +209,8 @@ public final class FileIndexFormat {
/** Reader for file index file. */
public static class Reader implements Closeable {
+ private static final Logger LOG =
LoggerFactory.getLogger(Reader.class);
+
private final SeekableInputStream seekableInputStream;
// Cache the index entries.
private final Map<String, Map<String, Pair<Long, Long>>> indexEntries
= new HashMap<>();
@@ -266,10 +271,20 @@ public final class FileIndexFormat {
}
public Set<FileIndexReader> readColumnIndex(String columnName) {
+ return readColumnIndex(columnName, false);
+ }
+
+ public Set<FileIndexReader> readColumnIndex(String columnName, boolean
ignoreUnknownType) {
return Optional.ofNullable(indexEntries.getOrDefault(columnName,
null))
.map(
f ->
f.entrySet().stream()
+ .filter(
+ entry ->
+ !ignoreUnknownType
+ ||
isKnownIndex(
+
columnName,
+
entry.getKey()))
.map(
entry ->
getFileIndexReader(
@@ -280,6 +295,17 @@ public final class FileIndexFormat {
.orElse(Collections.emptySet());
}
+ private boolean isKnownIndex(String columnName, String indexType) {
+ if (FileIndexerFactoryUtils.isRegistered(indexType)) {
+ return true;
+ }
+ LOG.warn(
+ "Ignoring unregistered file index type '{}' for column
'{}'; retaining all rows for this index.",
+ indexType,
+ columnName);
+ return false;
+ }
+
/** Returns the parsed index metadata without reading index payloads.
*/
public List<FileIndexMeta> indexMetas() {
List<FileIndexMeta> metas = new ArrayList<>();
diff --git
a/paimon-common/src/main/java/org/apache/paimon/fileindex/FileIndexPredicate.java
b/paimon-common/src/main/java/org/apache/paimon/fileindex/FileIndexPredicate.java
index 6874f8116d..57ea87b003 100644
---
a/paimon-common/src/main/java/org/apache/paimon/fileindex/FileIndexPredicate.java
+++
b/paimon-common/src/main/java/org/apache/paimon/fileindex/FileIndexPredicate.java
@@ -109,12 +109,17 @@ public class FileIndexPredicate implements Closeable {
}
public FileIndexResult evaluate(@Nullable Predicate predicate) {
+ return evaluate(predicate, false);
+ }
+
+ public FileIndexResult evaluate(@Nullable Predicate predicate, boolean
ignoreUnknownType) {
if (predicate == null) {
return REMAIN;
}
Set<String> requiredFieldNames = getRequiredNames(predicate);
Map<String, Collection<FileIndexReader>> indexReaders = new
HashMap<>();
- requiredFieldNames.forEach(name -> indexReaders.put(name,
reader.readColumnIndex(name)));
+ requiredFieldNames.forEach(
+ name -> indexReaders.put(name, reader.readColumnIndex(name,
ignoreUnknownType)));
FileIndexResult result = new
FileIndexPredicateTest(indexReaders).test(predicate);
if (!result.remain()) {
LOG.debug(
@@ -125,6 +130,11 @@ public class FileIndexPredicate implements Closeable {
}
public FileIndexResult evaluateTopN(@Nullable TopN topN, FileIndexResult
result) {
+ return evaluateTopN(topN, result, false);
+ }
+
+ public FileIndexResult evaluateTopN(
+ @Nullable TopN topN, FileIndexResult result, boolean
ignoreUnknownType) {
if (topN == null || !result.remain()) {
return result;
}
@@ -139,7 +149,7 @@ public class FileIndexPredicate implements Closeable {
List<SortValue> orders = topN.orders();
String requiredName = orders.get(0).field().name();
- Set<FileIndexReader> readers = reader.readColumnIndex(requiredName);
+ Set<FileIndexReader> readers = reader.readColumnIndex(requiredName,
ignoreUnknownType);
for (FileIndexReader reader : readers) {
FileIndexResult ret = reader.visitTopN(topN, result);
if (!REMAIN.equals(ret)) {
diff --git
a/paimon-common/src/main/java/org/apache/paimon/fileindex/FileIndexerFactoryUtils.java
b/paimon-common/src/main/java/org/apache/paimon/fileindex/FileIndexerFactoryUtils.java
index 29f922d077..944305178c 100644
---
a/paimon-common/src/main/java/org/apache/paimon/fileindex/FileIndexerFactoryUtils.java
+++
b/paimon-common/src/main/java/org/apache/paimon/fileindex/FileIndexerFactoryUtils.java
@@ -46,6 +46,10 @@ public class FileIndexerFactoryUtils {
}
}
+ public static boolean isRegistered(String type) {
+ return factories.containsKey(type);
+ }
+
public static FileIndexerFactory load(String type) {
FileIndexerFactory fileIndexerFactory = factories.get(type);
if (fileIndexerFactory == null) {
diff --git
a/paimon-common/src/test/java/org/apache/paimon/fileindex/UnknownFileIndexTest.java
b/paimon-common/src/test/java/org/apache/paimon/fileindex/UnknownFileIndexTest.java
new file mode 100644
index 0000000000..ac51ed5d2a
--- /dev/null
+++
b/paimon-common/src/test/java/org/apache/paimon/fileindex/UnknownFileIndexTest.java
@@ -0,0 +1,160 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.fileindex;
+
+import org.apache.paimon.fileindex.bitmap.BitmapFileIndex;
+import org.apache.paimon.fileindex.bitmap.BitmapIndexResult;
+import org.apache.paimon.options.Options;
+import org.apache.paimon.predicate.FieldRef;
+import org.apache.paimon.predicate.Predicate;
+import org.apache.paimon.predicate.PredicateBuilder;
+import org.apache.paimon.predicate.SortValue;
+import org.apache.paimon.predicate.TopN;
+import org.apache.paimon.types.DataTypes;
+import org.apache.paimon.types.RowType;
+import org.apache.paimon.utils.RoaringBitmap32;
+
+import org.apache.logging.log4j.Level;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.core.Logger;
+import org.apache.logging.log4j.core.appender.OutputStreamAppender;
+import org.apache.logging.log4j.core.layout.PatternLayout;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+
+import static org.apache.paimon.fileindex.FileIndexResult.REMAIN;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+/** Tests for reading containers with unregistered file index types. */
+class UnknownFileIndexTest {
+
+ private static final RowType ROW_TYPE = RowType.builder().field("a",
DataTypes.INT()).build();
+ private static final Predicate FILTER = new
PredicateBuilder(ROW_TYPE).equal(0, 1);
+
+ @ParameterizedTest
+ @ValueSource(ints = {1, 2})
+ void testUnknownIndexRetainsAllRowsAndWarns(int version) throws Exception {
+ ByteArrayOutputStream logs = new ByteArrayOutputStream();
+ OutputStreamAppender appender =
+ OutputStreamAppender.newBuilder()
+ .setName("unknown-file-index-test")
+ .setTarget(logs)
+
.setLayout(PatternLayout.newBuilder().withPattern("%level %msg%n").build())
+ .build();
+ Logger logger = (Logger)
LogManager.getLogger(FileIndexFormat.Reader.class.getName());
+ Level previousLevel = logger.getLevel();
+ appender.start();
+ logger.addAppender(appender);
+ logger.setLevel(Level.WARN);
+ try (FileIndexPredicate predicate = predicate(version, false, false)) {
+ assertThatThrownBy(() -> predicate.evaluate(FILTER))
+ .hasMessage("Can't find file index for type: unknown");
+ assertThatThrownBy(() -> predicate.evaluate(FILTER, false))
+ .hasMessage("Can't find file index for type: unknown");
+ assertThat(predicate.evaluate(FILTER, true)).isSameAs(REMAIN);
+ assertThat(new String(logs.toByteArray(), StandardCharsets.UTF_8))
+ .contains(
+ "WARN",
+ "Ignoring unregistered file index type 'unknown'
for column 'a'");
+ } finally {
+ logger.setLevel(previousLevel);
+ logger.removeAppender(appender);
+ appender.stop();
+ }
+ }
+
+ @ParameterizedTest
+ @ValueSource(ints = {1, 2})
+ void testRegisteredIndexStillFilters(int version) throws Exception {
+ try (FileIndexPredicate predicate = predicate(version, true, false)) {
+ BitmapIndexResult result = (BitmapIndexResult)
predicate.evaluate(FILTER, true);
+ assertThat(result.get()).isEqualTo(RoaringBitmap32.bitmapOf(0));
+ assertThat(
+ predicate
+ .evaluate(new
PredicateBuilder(ROW_TYPE).equal(0, 3), true)
+ .remain())
+ .isFalse();
+ Predicate or = PredicateBuilder.or(FILTER, new
PredicateBuilder(ROW_TYPE).equal(0, 2));
+ assertThat(((BitmapIndexResult) predicate.evaluate(or,
true)).get())
+ .isEqualTo(RoaringBitmap32.bitmapOf(0, 1));
+ }
+ }
+
+ @ParameterizedTest
+ @ValueSource(ints = {1, 2})
+ void testUnknownEmptyIndexIsIgnoredWhenEnabled(int version) throws
Exception {
+ try (FileIndexPredicate predicate = predicate(version, false, true)) {
+ assertThat(predicate.evaluate(FILTER).remain()).isFalse();
+ assertThat(predicate.evaluate(FILTER, true)).isSameAs(REMAIN);
+ }
+ }
+
+ @ParameterizedTest
+ @ValueSource(ints = {1, 2})
+ void testTopNRetainsAllRows(int version) throws Exception {
+ TopN topN =
+ new TopN(
+ new FieldRef(0, "a", DataTypes.INT()),
+ SortValue.SortDirection.ASCENDING,
+ SortValue.NullOrdering.NULLS_FIRST,
+ 1);
+ try (FileIndexPredicate predicate = predicate(version, false, false)) {
+ assertThatThrownBy(() -> predicate.evaluateTopN(topN, REMAIN))
+ .hasMessage("Can't find file index for type: unknown");
+ assertThat(predicate.evaluateTopN(topN, REMAIN,
true)).isSameAs(REMAIN);
+ }
+ }
+
+ @ParameterizedTest
+ @ValueSource(ints = {1, 2})
+ void testRegisteredIndexErrorsAreNotIgnored(int version) throws Exception {
+ ByteArrayOutputStream output = new ByteArrayOutputStream();
+ try (FileIndexFormat.Writer writer =
FileIndexFormat.createWriter(output, version)) {
+ writer.writeIndex("a", "bitmap", out -> out.write(42));
+ writer.finish();
+ }
+ try (FileIndexPredicate predicate =
+ new FileIndexPredicate(output.toByteArray(), ROW_TYPE)) {
+ assertThatThrownBy(() -> predicate.evaluate(FILTER, true))
+ .isInstanceOf(RuntimeException.class);
+ }
+ }
+
+ private static FileIndexPredicate predicate(int version, boolean bitmap,
boolean empty)
+ throws IOException {
+ ByteArrayOutputStream output = new ByteArrayOutputStream();
+ try (FileIndexFormat.Writer writer =
FileIndexFormat.createWriter(output, version)) {
+ writer.writeIndex("a", "unknown", empty ? null : out ->
out.write(42));
+ if (bitmap) {
+ FileIndexWriter index =
+ new BitmapFileIndex(DataTypes.INT(), new
Options()).createWriter();
+ index.writeRecord(1);
+ index.writeRecord(2);
+ writer.writeIndex("a", "bitmap", index::writeTo);
+ }
+ writer.finish();
+ }
+ return new FileIndexPredicate(output.toByteArray(), ROW_TYPE);
+ }
+}
diff --git
a/paimon-core/src/main/java/org/apache/paimon/AppendOnlyFileStore.java
b/paimon-core/src/main/java/org/apache/paimon/AppendOnlyFileStore.java
index b60e91906a..ac3a7e7bc9 100644
--- a/paimon-core/src/main/java/org/apache/paimon/AppendOnlyFileStore.java
+++ b/paimon-core/src/main/java/org/apache/paimon/AppendOnlyFileStore.java
@@ -173,6 +173,7 @@ public class AppendOnlyFileStore extends
AbstractFileStore<InternalRow> {
manifestFileFactory(),
options.scanManifestParallelism(),
options.fileIndexReadEnabled(),
+ options.fileIndexReadIgnoreUnknownType(),
options.deletionVectorsEnabled(),
options.dataEvolutionEnabled());
}
diff --git a/paimon-core/src/main/java/org/apache/paimon/KeyValueFileStore.java
b/paimon-core/src/main/java/org/apache/paimon/KeyValueFileStore.java
index 2f99d2367e..4b19e78eae 100644
--- a/paimon-core/src/main/java/org/apache/paimon/KeyValueFileStore.java
+++ b/paimon-core/src/main/java/org/apache/paimon/KeyValueFileStore.java
@@ -244,7 +244,8 @@ public class KeyValueFileStore extends
AbstractFileStore<KeyValue> {
options.deletionVectorsEnabled(),
options.mergeEngine(),
options.changelogProducer(),
- options.fileIndexReadEnabled() &&
options.deletionVectorsEnabled());
+ options.fileIndexReadEnabled() &&
options.deletionVectorsEnabled(),
+ options.fileIndexReadIgnoreUnknownType());
}
@Override
diff --git
a/paimon-core/src/main/java/org/apache/paimon/io/FileIndexEvaluator.java
b/paimon-core/src/main/java/org/apache/paimon/io/FileIndexEvaluator.java
index 9d88923f2f..bdd9de27ae 100644
--- a/paimon-core/src/main/java/org/apache/paimon/io/FileIndexEvaluator.java
+++ b/paimon-core/src/main/java/org/apache/paimon/io/FileIndexEvaluator.java
@@ -75,6 +75,31 @@ public class FileIndexEvaluator {
@Nullable DeletionVector dv,
long fileOffset)
throws IOException {
+ return evaluate(
+ fileIO,
+ dataSchema,
+ dataFilter,
+ topN,
+ limit,
+ dataFilePathFactory,
+ file,
+ dv,
+ fileOffset,
+ false);
+ }
+
+ public static FileIndexResult evaluate(
+ FileIO fileIO,
+ TableSchema dataSchema,
+ List<Predicate> dataFilter,
+ @Nullable TopN topN,
+ @Nullable Integer limit,
+ DataFilePathFactory dataFilePathFactory,
+ DataFileMeta file,
+ @Nullable DeletionVector dv,
+ long fileOffset,
+ boolean ignoreUnknownType)
+ throws IOException {
// File index selections use 32-bit positions. Fall back when they
cannot safely represent
// the file or its deletion vector.
if (file.rowCount() > RoaringBitmap32.MAX_VALUE || dv instanceof
Bitmap64DeletionVector) {
@@ -100,14 +125,14 @@ public class FileIndexEvaluator {
FileIndexResult result;
if (!isNullOrEmpty(dataFilter)) {
Predicate filter = PredicateBuilder.and(dataFilter.toArray(new
Predicate[0]));
- result = predicate.evaluate(filter);
+ result = predicate.evaluate(filter, ignoreUnknownType);
result = result.and(selection);
} else if (topN != null) {
// 1. TopN cannot work with filter, because a filter may not
completely filter out
// all records, any unfiltered records can affect the
calculation results of TopN
// 2. evaluateTopN with selection, because we must filter out
the data based on
// deletion vector before selecting TopN records.
- result = predicate.evaluateTopN(topN, selection);
+ result = predicate.evaluateTopN(topN, selection,
ignoreUnknownType);
} else {
return FileIndexResult.REMAIN;
}
diff --git
a/paimon-core/src/main/java/org/apache/paimon/operation/AppendOnlyFileStoreScan.java
b/paimon-core/src/main/java/org/apache/paimon/operation/AppendOnlyFileStoreScan.java
index af38f1d6c9..927ec69c26 100644
---
a/paimon-core/src/main/java/org/apache/paimon/operation/AppendOnlyFileStoreScan.java
+++
b/paimon-core/src/main/java/org/apache/paimon/operation/AppendOnlyFileStoreScan.java
@@ -47,6 +47,7 @@ public class AppendOnlyFileStoreScan extends
AbstractFileStoreScan {
private final SimpleStatsEvolutions simpleStatsEvolutions;
private final boolean fileIndexReadEnabled;
+ private final boolean fileIndexReadIgnoreUnknownType;
private final boolean deletionVectorsEnabled;
private final boolean dataEvolutionEnabled;
@@ -67,6 +68,7 @@ public class AppendOnlyFileStoreScan extends
AbstractFileStoreScan {
ManifestFile.Factory manifestFileFactory,
Integer scanManifestParallelism,
boolean fileIndexReadEnabled,
+ boolean fileIndexReadIgnoreUnknownType,
boolean deletionVectorsEnabled,
boolean dataEvolutionEnabled) {
super(
@@ -80,6 +82,7 @@ public class AppendOnlyFileStoreScan extends
AbstractFileStoreScan {
this.simpleStatsEvolutions =
new SimpleStatsEvolutions(sid ->
scanTableSchema(sid).fields(), schema.id());
this.fileIndexReadEnabled = fileIndexReadEnabled;
+ this.fileIndexReadIgnoreUnknownType = fileIndexReadIgnoreUnknownType;
this.deletionVectorsEnabled = deletionVectorsEnabled;
this.dataEvolutionEnabled = dataEvolutionEnabled;
}
@@ -180,7 +183,7 @@ public class AppendOnlyFileStoreScan extends
AbstractFileStoreScan {
try (FileIndexPredicate predicate =
new FileIndexPredicate(embeddedIndexBytes, dataRowType)) {
- return predicate.evaluate(dataPredicate).remain();
+ return predicate.evaluate(dataPredicate,
fileIndexReadIgnoreUnknownType).remain();
} catch (IOException e) {
throw new RuntimeException("Exception happens while checking
predicate.", e);
}
diff --git
a/paimon-core/src/main/java/org/apache/paimon/operation/DataEvolutionFileStoreScan.java
b/paimon-core/src/main/java/org/apache/paimon/operation/DataEvolutionFileStoreScan.java
index 442ad4c70d..8bc55a0e3d 100644
---
a/paimon-core/src/main/java/org/apache/paimon/operation/DataEvolutionFileStoreScan.java
+++
b/paimon-core/src/main/java/org/apache/paimon/operation/DataEvolutionFileStoreScan.java
@@ -97,6 +97,7 @@ public class DataEvolutionFileStoreScan extends
AppendOnlyFileStoreScan {
manifestFileFactory,
scanManifestParallelism,
false,
+ false,
deletionVectorsEnabled,
true);
this.deletionVectorsEnabled = deletionVectorsEnabled;
diff --git
a/paimon-core/src/main/java/org/apache/paimon/operation/DataEvolutionSplitRead.java
b/paimon-core/src/main/java/org/apache/paimon/operation/DataEvolutionSplitRead.java
index 6a1a2cc94d..ee6f4ab8a3 100644
---
a/paimon-core/src/main/java/org/apache/paimon/operation/DataEvolutionSplitRead.java
+++
b/paimon-core/src/main/java/org/apache/paimon/operation/DataEvolutionSplitRead.java
@@ -834,7 +834,8 @@ public class DataEvolutionSplitRead implements
SplitRead<InternalRow> {
dataFilePathFactory,
file,
dv,
- fileOffset);
+ fileOffset,
+ coreOptions.fileIndexReadIgnoreUnknownType());
if (!fileIndexResult.remain()) {
return new EmptyFileRecordReader<>();
}
@@ -1068,7 +1069,9 @@ public class DataEvolutionSplitRead implements
SplitRead<InternalRow> {
null,
dataFilePathFactory,
file,
- null);
+ null,
+ 0L,
+ coreOptions.fileIndexReadIgnoreUnknownType());
}
/**
@@ -1111,7 +1114,16 @@ public class DataEvolutionSplitRead implements
SplitRead<InternalRow> {
FileIndexResult result =
FileIndexEvaluator.evaluate(
- fileIO, dataSchema, dataFilters, null, null,
pathFactory, file, null);
+ fileIO,
+ dataSchema,
+ dataFilters,
+ null,
+ null,
+ pathFactory,
+ file,
+ null,
+ 0L,
+ coreOptions.fileIndexReadIgnoreUnknownType());
results.add(new FileIndexResultEntry(file, result));
if (!result.remain()) {
return results;
diff --git
a/paimon-core/src/main/java/org/apache/paimon/operation/KeyValueFileStoreScan.java
b/paimon-core/src/main/java/org/apache/paimon/operation/KeyValueFileStoreScan.java
index a3623c1b90..b2342e81bb 100644
---
a/paimon-core/src/main/java/org/apache/paimon/operation/KeyValueFileStoreScan.java
+++
b/paimon-core/src/main/java/org/apache/paimon/operation/KeyValueFileStoreScan.java
@@ -64,6 +64,7 @@ public class KeyValueFileStoreScan extends
AbstractFileStoreScan {
private final MergeEngine mergeEngine;
private final ChangelogProducer changelogProducer;
private final boolean fileIndexReadEnabled;
+ private final boolean fileIndexReadIgnoreUnknownType;
private Predicate keyFilter;
private Predicate valueFilter;
@@ -90,7 +91,8 @@ public class KeyValueFileStoreScan extends
AbstractFileStoreScan {
boolean deletionVectorsEnabled,
MergeEngine mergeEngine,
ChangelogProducer changelogProducer,
- boolean fileIndexReadEnabled) {
+ boolean fileIndexReadEnabled,
+ boolean fileIndexReadIgnoreUnknownType) {
super(
manifestsReader,
snapshotManager,
@@ -112,6 +114,7 @@ public class KeyValueFileStoreScan extends
AbstractFileStoreScan {
this.mergeEngine = mergeEngine;
this.changelogProducer = changelogProducer;
this.fileIndexReadEnabled = fileIndexReadEnabled;
+ this.fileIndexReadIgnoreUnknownType = fileIndexReadIgnoreUnknownType;
}
public KeyValueFileStoreScan withKeyFilter(Predicate predicate) {
@@ -186,7 +189,7 @@ public class KeyValueFileStoreScan extends
AbstractFileStoreScan {
id ->
fieldValueStatsConverters.tryDevolveFilter(
entry.file().schemaId(),
valueFilter));
- return predicate.evaluate(dataPredicate).remain();
+ return predicate.evaluate(dataPredicate,
fileIndexReadIgnoreUnknownType).remain();
} catch (IOException e) {
throw new RuntimeException("Exception happens while checking
fileIndex predicate.", e);
}
diff --git
a/paimon-core/src/main/java/org/apache/paimon/operation/RawFileSplitRead.java
b/paimon-core/src/main/java/org/apache/paimon/operation/RawFileSplitRead.java
index ea652001a0..e417ab87b9 100644
---
a/paimon-core/src/main/java/org/apache/paimon/operation/RawFileSplitRead.java
+++
b/paimon-core/src/main/java/org/apache/paimon/operation/RawFileSplitRead.java
@@ -86,6 +86,7 @@ public class RawFileSplitRead implements
SplitRead<InternalRow> {
private final FileStorePathFactory pathFactory;
private final Map<FormatKey, FormatReaderMapping> formatReaderMappings;
private final boolean fileIndexReadEnabled;
+ private final boolean fileIndexReadIgnoreUnknownType;
private final boolean rowTrackingEnabled;
private final boolean nestedFieldEnabled;
private final boolean ignoreCorruptFiles;
@@ -114,6 +115,7 @@ public class RawFileSplitRead implements
SplitRead<InternalRow> {
this.pathFactory = pathFactory;
this.formatReaderMappings = new HashMap<>();
this.fileIndexReadEnabled = coreOptions.fileIndexReadEnabled();
+ this.fileIndexReadIgnoreUnknownType =
coreOptions.fileIndexReadIgnoreUnknownType();
this.ignoreCorruptFiles = coreOptions.scanIgnoreCorruptFile();
this.ignoreLostFiles = coreOptions.scanIgnoreLostFile();
this.rowTrackingEnabled = coreOptions.rowTrackingEnabled();
@@ -335,7 +337,9 @@ public class RawFileSplitRead implements
SplitRead<InternalRow> {
formatReaderMapping.getLimit(),
dataFilePathFactory,
file,
- deletionVector);
+ deletionVector,
+ 0L,
+ fileIndexReadIgnoreUnknownType);
if (!fileIndexResult.remain()) {
return new EmptyFileRecordReader<>();
}
diff --git
a/paimon-core/src/test/java/org/apache/paimon/io/FileIndexEvaluatorTest.java
b/paimon-core/src/test/java/org/apache/paimon/io/FileIndexEvaluatorTest.java
index 5c1c4c79da..78b3e2d108 100644
--- a/paimon-core/src/test/java/org/apache/paimon/io/FileIndexEvaluatorTest.java
+++ b/paimon-core/src/test/java/org/apache/paimon/io/FileIndexEvaluatorTest.java
@@ -18,6 +18,7 @@
package org.apache.paimon.io;
+import org.apache.paimon.CoreOptions;
import org.apache.paimon.deletionvectors.Bitmap64DeletionVector;
import org.apache.paimon.deletionvectors.BitmapDeletionVector;
import org.apache.paimon.fileindex.FileIndexFormat;
@@ -26,7 +27,10 @@ import org.apache.paimon.fileindex.FileIndexWriter;
import org.apache.paimon.fileindex.bitmap.BitmapFileIndex;
import org.apache.paimon.fileindex.bitmap.BitmapFileIndexFactory;
import org.apache.paimon.fileindex.bitmap.BitmapIndexResult;
+import org.apache.paimon.fs.Path;
+import org.apache.paimon.fs.local.LocalFileIO;
import org.apache.paimon.options.Options;
+import org.apache.paimon.predicate.Predicate;
import org.apache.paimon.predicate.PredicateBuilder;
import org.apache.paimon.schema.TableSchema;
import org.apache.paimon.stats.SimpleStats;
@@ -37,16 +41,98 @@ import org.apache.paimon.types.RowType;
import org.apache.paimon.utils.RoaringBitmap32;
import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.CsvSource;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
+import java.io.OutputStream;
import java.util.Collections;
+import java.util.List;
import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
/** Tests for {@link FileIndexEvaluator}. */
public class FileIndexEvaluatorTest {
+ @TempDir java.nio.file.Path tempDir;
+
+ @ParameterizedTest
+ @CsvSource({"1,true", "1,false", "2,true", "2,false"})
+ void testIgnoreUnknownIndex(int version, boolean embedded) throws
Exception {
+ Options options = new Options();
+ assertThat(new
CoreOptions(options).fileIndexReadIgnoreUnknownType()).isFalse();
+ options.set(CoreOptions.FILE_INDEX_READ_IGNORE_UNKNOWN_TYPE, true);
+ ByteArrayOutputStream bytes = new ByteArrayOutputStream();
+ try (FileIndexFormat.Writer writer =
FileIndexFormat.createWriter(bytes, version)) {
+ writer.writeIndex("a", "unknown", out -> out.write(42));
+ writer.finish();
+ }
+ LocalFileIO fileIO = LocalFileIO.create();
+ DataFilePathFactory pathFactory =
+ new DataFilePathFactory(
+ new Path(tempDir.toUri()),
+ "avro",
+ "data-",
+ "changelog-",
+ false,
+ null,
+ null);
+ String indexName = "data.index";
+ if (!embedded) {
+ try (OutputStream out =
+ fileIO.newOutputStream(new Path(pathFactory.parent(),
indexName), false)) {
+ out.write(bytes.toByteArray());
+ }
+ }
+ DataFileMeta file =
+ DataFileMeta.forAppend(
+ "data.avro",
+ 0,
+ 2,
+ SimpleStats.EMPTY_STATS,
+ 0,
+ 0,
+ 0,
+ embedded ? Collections.emptyList() :
Collections.singletonList(indexName),
+ embedded ? bytes.toByteArray() : null,
+ null,
+ null,
+ null,
+ null,
+ null);
+ List<Predicate> filters =
+ Collections.singletonList(
+ new
PredicateBuilder(tableSchema().logicalRowType()).equal(0, 1));
+ assertThatThrownBy(
+ () ->
+ FileIndexEvaluator.evaluate(
+ fileIO,
+ tableSchema(),
+ filters,
+ null,
+ null,
+ pathFactory,
+ file,
+ null))
+ .hasMessage("Can't find file index for type: unknown");
+ assertThat(
+ FileIndexEvaluator.evaluate(
+ fileIO,
+ tableSchema(),
+ filters,
+ null,
+ null,
+ pathFactory,
+ file,
+ null,
+ 0L,
+ new
CoreOptions(options).fileIndexReadIgnoreUnknownType()))
+ .isSameAs(FileIndexResult.REMAIN);
+ }
+
@Test
public void testLimitIntersectsBitmapDeletionVector() throws Exception {
BitmapDeletionVector deletionVector = new BitmapDeletionVector();
diff --git
a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileTest.java
b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileTest.java
index 2eb0c1f4cb..e48a352da1 100644
--- a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileTest.java
+++ b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileTest.java
@@ -1770,6 +1770,7 @@ public class ManifestFileTest {
2,
false,
false,
+ false,
false);
scan.withRowRanges(Collections.singletonList(new Range(100, 100)));
fileIO.reset();
@@ -1828,6 +1829,7 @@ public class ManifestFileTest {
2,
false,
false,
+ false,
false);
scan.withBucket(1);
io.reset();
@@ -1892,6 +1894,7 @@ public class ManifestFileTest {
2,
false,
false,
+ false,
false);
io.reset();
List<ManifestEntry> actual = scan.readManifest(meta);
diff --git
a/paimon-core/src/test/java/org/apache/paimon/table/DataEvolutionFileIndexTest.java
b/paimon-core/src/test/java/org/apache/paimon/table/DataEvolutionFileIndexTest.java
index c060daf7b7..149ce51310 100644
---
a/paimon-core/src/test/java/org/apache/paimon/table/DataEvolutionFileIndexTest.java
+++
b/paimon-core/src/test/java/org/apache/paimon/table/DataEvolutionFileIndexTest.java
@@ -33,6 +33,7 @@ import
org.apache.paimon.deletionvectors.ApplyDeletionVectorReader;
import org.apache.paimon.deletionvectors.BitmapDeletionVector;
import org.apache.paimon.deletionvectors.DeletionVector;
import org.apache.paimon.deletionvectors.append.BaseAppendDeleteFileMaintainer;
+import org.apache.paimon.fileindex.FileIndexFormat;
import org.apache.paimon.fileindex.FileIndexOptions;
import org.apache.paimon.fileindex.bitmap.ApplyBitmapIndexRecordReader;
import org.apache.paimon.fileindex.bitmap.BitmapFileIndexFactory;
@@ -78,6 +79,7 @@ import org.junit.jupiter.params.provider.ValueSource;
import javax.annotation.Nullable;
+import java.io.OutputStream;
import java.lang.reflect.Field;
import java.util.ArrayList;
import java.util.Arrays;
@@ -92,6 +94,7 @@ import java.util.stream.Collectors;
import static org.apache.paimon.table.SpecialFields.rowTypeWithRowId;
import static org.apache.paimon.utils.DataEvolutionUtils.retrieveAnchorFile;
import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
/**
* Tests file index and predicate push down in {@link
@@ -1005,6 +1008,48 @@ public class DataEvolutionFileIndexTest extends
DataEvolutionTestBase {
assertThat(readWithFilter(table,
equalF1(MISSING_F1))).hasSize(ROW_COUNT);
}
+ @ParameterizedTest
+ @ValueSource(ints = {1, 2})
+ public void testIgnoreUnknownFileIndexInSingleFile(int version) throws
Exception {
+ Map<String, String> options = bloomOptions("f1", "1 B");
+ options.put(CoreOptions.FILE_INDEX_FORMAT_VERSION.key(),
Integer.toString(version));
+ FileStoreTable table = createTable("unknown_single_file", options);
+ writeAllColumns(table, ROW_COUNT);
+
+ DataSplit split = (DataSplit)
table.newReadBuilder().newScan().plan().splits().get(0);
+ assertThat(split.dataFiles()).hasSize(1);
+ DataFileMeta file = split.dataFiles().get(0);
+ DataFilePathFactory pathFactory =
+ table.store()
+ .pathFactory()
+ .createDataFilePathFactory(split.partition(),
split.bucket());
+ List<String> indexes =
+ file.extraFiles().stream()
+ .filter(name ->
name.endsWith(DataFilePathFactory.INDEX_PATH_SUFFIX))
+ .collect(Collectors.toList());
+ assertThat(indexes).hasSize(1);
+ // Replace the sidecar with an index produced by a plugin unavailable
to this reader.
+ try (OutputStream out =
+ table.fileIO()
+ .newOutputStream(
+
pathFactory.toAlignedPath(indexes.get(0), file), true);
+ FileIndexFormat.Writer writer =
FileIndexFormat.createWriter(out, version)) {
+ writer.writeIndex("f1", "unknown", payload -> payload.write(42));
+ writer.finish();
+ }
+
+ assertThatThrownBy(() -> readWithFilter(table, equalF1(MISSING_F1)))
+ .hasMessage("Can't find file index for type: unknown");
+ catalog.alterTable(
+ identifier(table.name()),
+ SchemaChange.setOption(
+ CoreOptions.FILE_INDEX_READ_IGNORE_UNKNOWN_TYPE.key(),
"true"),
+ false);
+ assertThat(readWithFilter(table,
equalF1(MISSING_F1))).hasSize(ROW_COUNT);
+ assertThat(query(table, equalF1(MISSING_F1))).isEmpty();
+ assertRow(assertSingleRow(query(table, equalF1(f1(50)))), 50);
+ }
+
@Test
public void testQueryWithFileIndexReadDisabled() throws Exception {
Map<String, String> options = bloomOptions("f1", "1 B");
