This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new 36281f836b [lumina] Reject non-positive encoding.pq.m at index build
time (#9823)
36281f836b is described below
commit 36281f836bf2a1194fd74a912289039551c26527
Author: jackylee <[email protected]>
AuthorDate: Tue Sep 22 11:40:35 2026 +0800
[lumina] Reject non-positive encoding.pq.m at index build time (#9823)
---
.../index/LuminaVectorGlobalIndexWriter.java | 2 +-
.../lumina/index/LuminaVectorIndexOptions.java | 33 +++++++++++++++++++++-
.../lumina/index/LuminaVectorOptionsTest.java | 29 +++++++++++++++++++
3 files changed, 62 insertions(+), 2 deletions(-)
diff --git
a/paimon-lumina/src/main/java/org/apache/paimon/lumina/index/LuminaVectorGlobalIndexWriter.java
b/paimon-lumina/src/main/java/org/apache/paimon/lumina/index/LuminaVectorGlobalIndexWriter.java
index 0613405f46..f87761a113 100644
---
a/paimon-lumina/src/main/java/org/apache/paimon/lumina/index/LuminaVectorGlobalIndexWriter.java
+++
b/paimon-lumina/src/main/java/org/apache/paimon/lumina/index/LuminaVectorGlobalIndexWriter.java
@@ -103,7 +103,7 @@ public class LuminaVectorGlobalIndexWriter implements
GlobalIndexSingleColumnWri
this.fileWriter = fileWriter;
this.options = options;
this.dim = validateAndResolveDimension(fieldType, options);
- this.luminaOptions = options.toLuminaOptions(dim);
+ this.luminaOptions = options.toBuildOptions(dim);
this.count = 0;
this.closed = false;
this.recordSizeInBytes = checkedRecordSize(dim, IO_BUFFER_SIZE);
diff --git
a/paimon-lumina/src/main/java/org/apache/paimon/lumina/index/LuminaVectorIndexOptions.java
b/paimon-lumina/src/main/java/org/apache/paimon/lumina/index/LuminaVectorIndexOptions.java
index a7e8facb18..92727fd7f4 100644
---
a/paimon-lumina/src/main/java/org/apache/paimon/lumina/index/LuminaVectorIndexOptions.java
+++
b/paimon-lumina/src/main/java/org/apache/paimon/lumina/index/LuminaVectorIndexOptions.java
@@ -196,6 +196,35 @@ public class LuminaVectorIndexOptions {
return result;
}
+ /**
+ * Returns the native options an index build runs with, and rejects the
ones the native builder
+ * cannot use. Only the build path may reject: a search resolves its
options with the index
+ * metadata layered on top of these, so a value that is stale or invalid
on the table is
+ * overridden there and must not fail the read.
+ */
+ public Map<String, String> toBuildOptions(int dimension) {
+ Map<String, String> result = toLuminaOptions(dimension);
+ validateBuildOptions(result);
+ return result;
+ }
+
+ /**
+ * Lumina's QuantizerTrainer rejects a non-positive {@code numChunks} —
{@code "Invalid
+ * numChunks [%d] for PQ encoding, must be > 0 and <= dimension [%d]."} in
liblumina — but only
+ * once it is reached, which is after a build has spilled every vector of
the shard to disk.
+ * Rejecting the option up front names the Paimon key instead.
+ */
+ private static void validateBuildOptions(Map<String, String> opts) {
+ String encoding = opts.get(toLuminaKey(ENCODING_TYPE));
+ if (!"pq".equalsIgnoreCase(encoding)) {
+ return;
+ }
+ String pqMStr = opts.get(toLuminaKey(ENCODING_PQ_M));
+ if (pqMStr != null) {
+ validatePositive(Integer.parseInt(pqMStr), ENCODING_PQ_M.key());
+ }
+ }
+
public int dimension() {
return dimension;
}
@@ -283,7 +312,9 @@ public class LuminaVectorIndexOptions {
/**
* Ensures {@code encoding.pq.m} does not exceed the vector dimension.
Lumina's QuantizerTrainer
- * requires numChunks (pq.m) to be > 0 and ≤ dimension.
+ * requires numChunks (pq.m) to be > 0 and ≤ dimension; the upper
bound is capped here
+ * because it depends on the dimension, and the lower bound is checked by
{@link
+ * #toBuildOptions} because only a build can act on it.
*/
private static void capPqM(Map<String, String> opts, int dimension) {
String encoding = opts.get(toLuminaKey(ENCODING_TYPE));
diff --git
a/paimon-lumina/src/test/java/org/apache/paimon/lumina/index/LuminaVectorOptionsTest.java
b/paimon-lumina/src/test/java/org/apache/paimon/lumina/index/LuminaVectorOptionsTest.java
index 7403d41dd3..1fefd0327d 100644
---
a/paimon-lumina/src/test/java/org/apache/paimon/lumina/index/LuminaVectorOptionsTest.java
+++
b/paimon-lumina/src/test/java/org/apache/paimon/lumina/index/LuminaVectorOptionsTest.java
@@ -27,6 +27,7 @@ import java.util.Map;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatCode;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
/** Tests for Lumina vector options. */
public class LuminaVectorOptionsTest {
@@ -148,6 +149,34 @@ public class LuminaVectorOptionsTest {
.containsEntry("distance.metric", "l2");
}
+ @Test
+ public void testEncodingPqMBounds() {
+ // capPqM documents "> 0 and <= dimension" but enforced only the upper
bound, so pq.m = 0
+ // was handed to the native trainer instead of being reported against
its own key.
+ Map<String, String> zeroPqM = new HashMap<>();
+ zeroPqM.put("lumina.index.dimension", "128");
+ zeroPqM.put("lumina.encoding.type", "pq");
+ zeroPqM.put("lumina.encoding.pq.m", "0");
+ LuminaVectorIndexOptions zeroOptions =
+ new LuminaVectorIndexOptions(Options.fromMap(zeroPqM));
+
+ assertThatThrownBy(() -> zeroOptions.toBuildOptions(128))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining("lumina.encoding.pq.m");
+
+ // Only a build may reject: a search layers the index metadata over
these options, so the
+ // stale table value is overridden there and must not fail the read.
+
assertThatCode(zeroOptions::toLuminaOptions).doesNotThrowAnyException();
+
+ // The upper bound keeps clamping to the dimension rather than
throwing, on both paths.
+ Map<String, String> oversizedPqM = new HashMap<>(zeroPqM);
+ oversizedPqM.put("lumina.encoding.pq.m", "256");
+ LuminaVectorIndexOptions oversizedOptions =
+ new LuminaVectorIndexOptions(Options.fromMap(oversizedPqM));
+
assertThat(oversizedOptions.toLuminaOptions()).containsEntry("encoding.pq.m",
"128");
+
assertThat(oversizedOptions.toBuildOptions(128)).containsEntry("encoding.pq.m",
"128");
+ }
+
/** Builds the native lumina meta map (what gets serialized into the index
file) for a field. */
private static Map<String, String> metaFor(String fieldName, Map<String,
String> tableOptions) {
Options resolved =
