This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new 9af77b0a7c [lumina] Support per-field vector index options (#8260)
9af77b0a7c is described below
commit 9af77b0a7c43d120c69784fac8a42909d4e9c58f
Author: jerry <[email protected]>
AuthorDate: Wed Jun 17 20:20:08 2026 +0800
[lumina] Support per-field vector index options (#8260)
`paimon-lumina` only allowed Lumina vector index options to be
configured globally via `lumina.*` table options. A table with multiple
vector columns could therefore not configure each column independently
(e.g. a different `distance.metric` or `index.dimension`).
This PR adds per-field configuration, **following the same convention as
#8239** (`paimon-vector-index`).
---
.../index/LuminaVectorGlobalIndexerFactory.java | 3 +-
.../lumina/index/LuminaVectorIndexOptions.java | 67 +++++++++++++++++++
.../lumina/index/LuminaVectorOptionsTest.java | 75 ++++++++++++++++++++++
3 files changed, 144 insertions(+), 1 deletion(-)
diff --git
a/paimon-lumina/src/main/java/org/apache/paimon/lumina/index/LuminaVectorGlobalIndexerFactory.java
b/paimon-lumina/src/main/java/org/apache/paimon/lumina/index/LuminaVectorGlobalIndexerFactory.java
index 7638fa6098..7d9c062feb 100644
---
a/paimon-lumina/src/main/java/org/apache/paimon/lumina/index/LuminaVectorGlobalIndexerFactory.java
+++
b/paimon-lumina/src/main/java/org/apache/paimon/lumina/index/LuminaVectorGlobalIndexerFactory.java
@@ -35,6 +35,7 @@ public class LuminaVectorGlobalIndexerFactory implements
GlobalIndexerFactory {
@Override
public GlobalIndexer create(DataField field, Options options) {
- return new LuminaVectorGlobalIndexer(field.type(), options);
+ Options fieldOptions =
LuminaVectorIndexOptions.resolveFieldOptions(field.name(), options);
+ return new LuminaVectorGlobalIndexer(field.type(), fieldOptions);
}
}
diff --git
a/paimon-lumina/src/main/java/org/apache/paimon/lumina/index/LuminaVectorIndexOptions.java
b/paimon-lumina/src/main/java/org/apache/paimon/lumina/index/LuminaVectorIndexOptions.java
index 80c6b8e28e..843e5c3813 100644
---
a/paimon-lumina/src/main/java/org/apache/paimon/lumina/index/LuminaVectorIndexOptions.java
+++
b/paimon-lumina/src/main/java/org/apache/paimon/lumina/index/LuminaVectorIndexOptions.java
@@ -18,14 +18,17 @@
package org.apache.paimon.lumina.index;
+import org.apache.paimon.CoreOptions;
import org.apache.paimon.options.ConfigOption;
import org.apache.paimon.options.ConfigOptions;
import org.apache.paimon.options.Options;
import java.util.Arrays;
+import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
+import java.util.Set;
/** Options for the Lumina vector index. */
public class LuminaVectorIndexOptions {
@@ -124,6 +127,54 @@ public class LuminaVectorIndexOptions {
this.luminaOptions = buildLuminaOptions(options);
}
+ /**
+ * Resolves per-field Lumina options for {@code fieldName} into an
effective {@link Options}.
+ *
+ * <p>Following the convention shared with {@code paimon-vector-index} (PR
#8239), a field-level
+ * option is written {@code fields.<fieldName>.<option>} — <b>without</b>
the {@code lumina.}
+ * index-type prefix — and overrides the column-agnostic {@code
lumina.<option>} for that field
+ * only. For example {@code fields.embed.distance.metric} overrides {@code
+ * lumina.distance.metric} for column {@code embed}. The {@code <option>}
suffix is exactly the
+ * key used after {@code lumina.} at the table level.
+ *
+ * <p>Only recognized Lumina options (the keys in {@code
FIELD_OVERRIDABLE_KEYS}) are accepted;
+ * any other {@code fields.<fieldName>.*} key (e.g. a merge/aggregation
option) is left
+ * untouched, mirroring how {@code paimon-vector-index} ignores keys it
does not recognize.
+ *
+ * <p>Each recognized field option is flattened back to its plain {@code
lumina.*} form, so the
+ * rest of this class still sees only {@code lumina.*} keys and the
metadata produced from these
+ * options keeps the exact same native-key shape as before (no {@code
fields.*} keys ever reach
+ * the meta) — only the resolved value changes. This is what lets the
reader consume the meta
+ * unchanged.
+ *
+ * <p>The same resolution runs on both the write and read paths, since
both build the indexer
+ * through {@link LuminaVectorGlobalIndexerFactory#create}.
+ */
+ public static Options resolveFieldOptions(String fieldName, Options
options) {
+ String fieldPrefix = CoreOptions.FIELDS_PREFIX + "." + fieldName + ".";
+ Map<String, String> tableOptions = options.toMap();
+ Map<String, String> result = new LinkedHashMap<>();
+ // Base: table-level options, including the column-agnostic lumina.*
options. Drop all
+ // fields.* keys; this field's recognized options are re-added below
as lumina.* keys.
+ for (Map.Entry<String, String> entry : tableOptions.entrySet()) {
+ if (!entry.getKey().startsWith(CoreOptions.FIELDS_PREFIX + ".")) {
+ result.put(entry.getKey(), entry.getValue());
+ }
+ }
+ // Overlay this field's options. fields.<field>.<option> (no lumina.
prefix) overrides the
+ // column-agnostic lumina.<option>. Only recognized Lumina options are
taken.
+ for (Map.Entry<String, String> entry : tableOptions.entrySet()) {
+ String key = entry.getKey();
+ if (key.startsWith(fieldPrefix)) {
+ String option = key.substring(fieldPrefix.length());
+ if (FIELD_OVERRIDABLE_KEYS.contains(option)) {
+ result.put(LUMINA_PREFIX + option, entry.getValue());
+ }
+ }
+ }
+ return Options.fromMap(result);
+ }
+
/**
* Returns all {@code lumina.*} options with the prefix stripped,
producing native Lumina keys.
* For example, {@code lumina.diskann.build.ef_construction} becomes {@code
@@ -186,6 +237,22 @@ public class LuminaVectorIndexOptions {
ENCODING_PQ_M,
SEARCH_PARALLEL_NUMBER);
+ /**
+ * Native Lumina keys (the {@code lumina.} prefix stripped) that may be
overridden per field via
+ * {@code fields.<fieldName>.<key>}. Any {@code fields.<fieldName>.*} key
outside this set is
+ * ignored, so unrelated per-field options do not leak into the Lumina
index metadata.
+ */
+ private static final Set<String> FIELD_OVERRIDABLE_KEYS =
buildFieldOverridableKeys();
+
+ private static Set<String> buildFieldOverridableKeys() {
+ Set<String> keys = new HashSet<>();
+ for (ConfigOption<?> opt : ALL_OPTIONS) {
+ keys.add(toLuminaKey(opt));
+ }
+ keys.add(toLuminaKey(DISKANN_SEARCH_LIST_SIZE));
+ return keys;
+ }
+
/**
* Builds native Lumina options by first populating all known
ConfigOptions (with defaults) and
* then overlaying any user-specified {@code lumina.*} options. This
ensures that required keys
diff --git
a/paimon-lumina/src/test/java/org/apache/paimon/lumina/index/LuminaVectorOptionsTest.java
b/paimon-lumina/src/test/java/org/apache/paimon/lumina/index/LuminaVectorOptionsTest.java
index 7ea5088780..326b89fb9b 100644
---
a/paimon-lumina/src/test/java/org/apache/paimon/lumina/index/LuminaVectorOptionsTest.java
+++
b/paimon-lumina/src/test/java/org/apache/paimon/lumina/index/LuminaVectorOptionsTest.java
@@ -18,6 +18,8 @@
package org.apache.paimon.lumina.index;
+import org.apache.paimon.options.Options;
+
import org.junit.jupiter.api.Test;
import java.util.HashMap;
@@ -49,4 +51,77 @@ public class LuminaVectorOptionsTest {
.containsEntry("index.dimension", "4")
.containsEntry("hnsw.ef_search", "128");
}
+
+ @Test
+ public void testFieldOptionsOverrideGlobal() {
+ Map<String, String> tableOptions = new HashMap<>();
+ tableOptions.put("lumina.distance.metric", "l2");
+ tableOptions.put("lumina.index.dimension", "128");
+ // Field-level keys follow the #8239 convention:
fields.<field>.<option>, no lumina. prefix.
+ tableOptions.put("fields.embed.distance.metric", "inner_product");
+ tableOptions.put("fields.embed.index.dimension", "256");
+
+ Map<String, String> meta = metaFor("embed", tableOptions);
+
+ // The per-field value wins over the global one.
+ assertThat(meta)
+ .containsEntry("distance.metric", "inner_product")
+ .containsEntry("index.dimension", "256");
+ // The meta keeps the native key shape; no fields.* keys ever leak
into it.
+ assertThat(meta.keySet()).noneMatch(k -> k.startsWith("fields"));
+ }
+
+ @Test
+ public void testForeignFieldOptionsAreIgnored() {
+ Map<String, String> globalOnly = new HashMap<>();
+ globalOnly.put("lumina.distance.metric", "l2");
+ globalOnly.put("lumina.index.dimension", "128");
+
+ Map<String, String> withForeign = new HashMap<>(globalOnly);
+ // A per-field option that is not a recognized Lumina key (e.g. a
merge/agg option) must be
+ // ignored, never flattened into the index meta as a bogus native key.
+ withForeign.put("fields.embed.aggregate-function", "sum");
+
+ Map<String, String> meta = metaFor("embed", withForeign);
+ assertThat(meta).isEqualTo(metaFor("embed", globalOnly));
+ assertThat(meta).doesNotContainKey("aggregate-function");
+ }
+
+ @Test
+ public void testFieldOptionsForOtherFieldDoNotAffectMeta() {
+ Map<String, String> globalOnly = new HashMap<>();
+ globalOnly.put("lumina.distance.metric", "l2");
+ globalOnly.put("lumina.index.dimension", "128");
+
+ Map<String, String> withOtherField = new HashMap<>(globalOnly);
+ // Per-field config for a different column must not influence "embed".
+ withOtherField.put("fields.other.distance.metric", "inner_product");
+ withOtherField.put("fields.other.index.dimension", "256");
+
+ // The meta for "embed" is byte-for-byte the same as before per-field
support existed.
+ assertThat(metaFor("embed",
withOtherField)).isEqualTo(metaFor("embed", globalOnly));
+ }
+
+ @Test
+ public void testResolvedFieldMetaIsReadable() {
+ Map<String, String> tableOptions = new HashMap<>();
+ tableOptions.put("lumina.distance.metric", "l2");
+ tableOptions.put("lumina.index.dimension", "128");
+ tableOptions.put("fields.embed.distance.metric", "inner_product");
+ tableOptions.put("fields.embed.index.dimension", "256");
+
+ // The reader reconstructs everything it needs from the serialized
meta.
+ LuminaIndexMeta meta = new LuminaIndexMeta(metaFor("embed",
tableOptions));
+ assertThat(meta.dim()).isEqualTo(256);
+ assertThat(meta.distanceMetric()).isEqualTo("inner_product");
+ assertThat(meta.metric()).isEqualTo(LuminaVectorMetric.INNER_PRODUCT);
+ }
+
+ /** Builds the native lumina meta map (what gets serialized into the index
file) for a field. */
+ private static Map<String, String> metaFor(String fieldName, Map<String,
String> tableOptions) {
+ Options resolved =
+ LuminaVectorIndexOptions.resolveFieldOptions(
+ fieldName, Options.fromMap(tableOptions));
+ return new LuminaVectorIndexOptions(resolved).toLuminaOptions();
+ }
}
