This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new c23edd8dbd [vector] Upgrade paimon-vector-index to 0.5.0 and support
new build options (#10080)
c23edd8dbd is described below
commit c23edd8dbdf27c13784f80d109a0785245d0596d
Author: jerry <[email protected]>
AuthorDate: Wed Sep 23 11:26:56 2026 +0800
[vector] Upgrade paimon-vector-index to 0.5.0 and support new build options
(#10080)
---
docs/docs/multimodal-table/global-index/vector.mdx | 30 ++++-
.../java/org/apache/paimon/options/Options.java | 49 ++++++--
.../flink/globalindex/GenericIndexTopoBuilder.java | 2 +-
paimon-python/dev/requirements-dev.txt | 2 +-
paimon-python/dev/run_mixed_tests.sh | 6 +-
.../pypaimon/globalindex/create_global_index.py | 4 +-
.../vindex/vindex_vector_global_index_reader.py | 2 +-
.../vindex/vindex_vector_index_writer.py | 120 ++++++++++++++----
.../pypaimon/tests/global_index_build_test.py | 78 +++++++++++-
paimon-python/setup.py | 2 +-
paimon-vector/pom.xml | 2 +-
.../index/NativeVectorGlobalIndexerFactory.java | 137 ++++++++++++++++-----
.../NativeVectorGlobalIndexerFactoryTest.java | 133 +++++++++++++++++++-
13 files changed, 489 insertions(+), 78 deletions(-)
diff --git a/docs/docs/multimodal-table/global-index/vector.mdx
b/docs/docs/multimodal-table/global-index/vector.mdx
index c17170e9f9..c191389b22 100644
--- a/docs/docs/multimodal-table/global-index/vector.mdx
+++ b/docs/docs/multimodal-table/global-index/vector.mdx
@@ -73,7 +73,7 @@ The build and search examples below use its three-dimensional
`embedding` column
For this small dataset, use `ivf-flat` with one cluster; production workloads
should
choose an index type and cluster count for their own data.
-For Python, install `pypaimon[vindex]` (or its matching `paimon-vindex==0.4.0`
+For Python, install `pypaimon[vindex]` (or its matching `paimon-vindex==0.5.0`
dependency) before building or querying native vector indexes.
<Tabs groupId="vector-build">
@@ -335,15 +335,43 @@ Supported paimon-vindex options:
|---|---|---|
| `<index-type>.dimension` | `128` | Vector dimension for `ARRAY<FLOAT>`
columns. Ignored for `VECTOR<FLOAT, n>` columns. |
| `<index-type>.distance.metric` | `inner_product` | Distance metric.
Supported values: `l2`, `cosine`, `inner_product`. |
+| `<index-type>.ivf.coarse-assignment` | `auto` | Build-time list assignment
for all IVF types. `auto` uses Vamana when `dimension × nlist ≥ 1,000,000` and
exact assignment otherwise; `exact` always uses exact assignment. |
+| `<index-type>.ivf.train.max-points-per-centroid` | `256` | Positive coarse
K-means training limit for all IVF types: at most `nlist × value` vectors. |
| `<index-type>.train.sample-ratio` | `1.0` | Ratio of vectors sampled for
native index training. Must be greater than `0` and less than or equal to `1`.
Lower values reduce training memory and build cost, but may reduce index
quality. |
| `<index-type>.nlist` | Automatic | Number of clusters for the four IVF
types. When omitted, paimon-vindex resolves it from the shard's non-null vector
count. |
| `<index-type>.pq.code-ratio` | `0.0625` | Relative PQ-code budget for
`ivf-pq` and `diskann`. |
| `<index-type>.pq.m` | Automatic | Expert override for the PQ sub-vector
count used by `ivf-pq` and `diskann`. |
+| `ivf-pq.ivf.pq-encoding` | `auto` | IVF-PQ build-time encoding. `auto`
selects an accelerated backend when supported; `canonical` uses the canonical
encoder. |
+| `<index-type>.pq.train.max-points-per-centroid` | `256` | Positive PQ
training limit for `ivf-pq` and `diskann`: at most `2^pq.bits × value` vectors
per subquantizer. |
| `ivf-pq.pq.use-opq` | Automatic | Explicitly enables or disables OPQ.
Without an explicit value, a `target-recall` of at least `0.9` enables it. |
| `ivf-rq.rq.bits` | `4` | Persisted IVF-RQ residual width. Supported values
are `1` through `8`; changing it requires rebuilding the index. |
| `<index-type>.target-recall` | Not set | Build-policy hint used by `ivf-pq`
and `diskann`. Validate the resulting recall on held-out queries. |
| `<index-type>.max-bytes-per-vector` | Not set | Storage objective and
conservative preflight bound for `ivf-pq`, `ivf-rq`, and `diskann`. |
+Use the index-prefixed keys in table properties, SQL procedure options, and
Python
+build options. For example, the following IVF-PQ build selects the non-default
+exact and canonical paths and lowers both training limits from `256` to `128`:
+
+```sql
+CALL sys.create_global_index(
+ table => 'db.model_embeddings',
+ index_column => 'embedding',
+ index_type => 'ivf-pq',
+ options =>
'ivf-pq.ivf.coarse-assignment=exact,ivf-pq.ivf.pq-encoding=canonical,ivf-pq.ivf.train.max-points-per-centroid=128,ivf-pq.pq.train.max-points-per-centroid=128'
+);
+```
+
+Omit these options to retain the defaults in the table above. The native names
+`ivf.coarse-assignment`, `ivf.pq-encoding`,
+`ivf.train.max-points-per-centroid`, and `pq.train.max-points-per-centroid` are
+also accepted directly; prefer index-prefixed names in table properties to
avoid
+sharing a setting across index types. Use `fields.<field-name>.<option>` to
+override a table property for one vector column. Explicit procedure or Python
+options override table properties; within either source, field-prefixed options
+override index-prefixed options, which override bare native names. An
+inapplicable stored table property is ignored, while an explicitly supplied
+option that does not apply to the selected index type is rejected.
+
Additional paimon-vindex DiskANN build options:
| Option | Default | Description |
diff --git a/paimon-api/src/main/java/org/apache/paimon/options/Options.java
b/paimon-api/src/main/java/org/apache/paimon/options/Options.java
index 178adc7545..523c7412ed 100644
--- a/paimon-api/src/main/java/org/apache/paimon/options/Options.java
+++ b/paimon-api/src/main/java/org/apache/paimon/options/Options.java
@@ -51,27 +51,38 @@ public class Options implements Serializable {
/** Stores the concrete key/value pairs of this configuration object. */
private final HashMap<String, String> data;
+ /** Options overriding the base map through setters or the two-map
constructor. */
+ private HashMap<String, String> dynamicOptions;
+
/** Creates a new empty configuration. */
public Options() {
this.data = new HashMap<>();
+ this.dynamicOptions = new HashMap<>();
}
/** Creates a new configuration that is initialized with the options of
the given map. */
public Options(Map<String, String> map) {
this();
- map.forEach(this::setString);
+ map.forEach(this.data::put);
}
/** Creates a new configuration that is initialized with the options of
the given two maps. */
public Options(Map<String, String> map1, Map<String, String> map2) {
- this();
- map1.forEach(this::setString);
- map2.forEach(this::setString);
+ this(map1);
+ map2.forEach(this.data::put);
+ this.dynamicOptions.putAll(map2);
+ }
+
+ /** Merges a base map while preserving which options dynamically override
it. */
+ public Options(Map<String, String> map, Options options) {
+ this(map, options.toMap());
+ this.dynamicOptions.clear();
+ this.dynamicOptions.putAll(options.dynamicOptions());
}
public Options(Iterable<Map.Entry<String, String>> map) {
this();
- map.forEach(entry -> setString(entry.getKey(), entry.getValue()));
+ map.forEach(entry -> data.put(entry.getKey(), entry.getValue()));
}
public static Options fromMap(Map<String, String> map) {
@@ -86,10 +97,12 @@ public class Options implements Serializable {
*/
public synchronized void setString(String key, String value) {
data.put(key, value);
+ setDynamicOption(key, value);
}
public synchronized void set(String key, String value) {
data.put(key, value);
+ setDynamicOption(key, value);
}
public synchronized <T> Options set(ConfigOption<T> option, T value) {
@@ -150,8 +163,21 @@ public class Options implements Serializable {
return data;
}
+ /** Returns options supplied as dynamic overrides to the base map. */
+ public synchronized Map<String, String> dynamicOptions() {
+ Map<String, String> result = new HashMap<>();
+ if (dynamicOptions != null) {
+ dynamicOptions.keySet().stream()
+ .filter(data::containsKey)
+ .forEach(key -> result.put(key, data.get(key)));
+ }
+ return result;
+ }
+
public synchronized Options removePrefix(String prefix) {
- return new Options(convertToPropertiesPrefixKey(data, prefix));
+ return new Options(
+ convertToPropertiesPrefixKey(data, prefix),
+ convertToPropertiesPrefixKey(dynamicOptions(), prefix));
}
public synchronized String remove(String key) {
@@ -232,8 +258,17 @@ public class Options implements Serializable {
if (canBePrefixMap) {
removePrefixMap(this.data, key);
}
- this.data.put(key, OptionsUtils.convertToString(value));
+ String stringValue = OptionsUtils.convertToString(value);
+ this.data.put(key, stringValue);
+ setDynamicOption(key, stringValue);
+ }
+ }
+
+ private void setDynamicOption(String key, String value) {
+ if (dynamicOptions == null) {
+ dynamicOptions = new HashMap<>();
}
+ dynamicOptions.put(key, value);
}
private Optional<Object> getRawValueFromOption(ConfigOption<?>
configOption) {
diff --git
a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/globalindex/GenericIndexTopoBuilder.java
b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/globalindex/GenericIndexTopoBuilder.java
index 4740a3ea80..bb14cefa43 100644
---
a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/globalindex/GenericIndexTopoBuilder.java
+++
b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/globalindex/GenericIndexTopoBuilder.java
@@ -227,7 +227,7 @@ public class GenericIndexTopoBuilder {
readColumns.add(SpecialFields.ROW_ID.name());
RowType projectedRowType =
SpecialFields.rowTypeWithRowId(rowType).project(readColumns);
- Options mergedOptions = new Options(table.options(),
userOptions.toMap());
+ Options mergedOptions = new Options(table.options(), userOptions);
byte[] sourceMeta =
new
DataEvolutionIndexSourceMeta(scanResult.scanSnapshotId()).serialize();
diff --git a/paimon-python/dev/requirements-dev.txt
b/paimon-python/dev/requirements-dev.txt
index a122dd0bbe..711b0ee62a 100644
--- a/paimon-python/dev/requirements-dev.txt
+++ b/paimon-python/dev/requirements-dev.txt
@@ -36,6 +36,6 @@ datafusion>=52; python_version >= "3.10"
# Lumina vector search (optional, for lumina index tests)
lumina-data>=0.1.0
# paimon-vindex vector search (optional, for vindex index tests)
-paimon-vindex==0.4.0; python_version >= "3.9"
+paimon-vindex==0.5.0; python_version >= "3.9"
# paimon-ftindex full-text search (optional, for full-text index tests)
paimon-ftindex==0.1.0; python_version >= "3.8"
diff --git a/paimon-python/dev/run_mixed_tests.sh
b/paimon-python/dev/run_mixed_tests.sh
index 54fef2a4e1..319267b663 100755
--- a/paimon-python/dev/run_mixed_tests.sh
+++ b/paimon-python/dev/run_mixed_tests.sh
@@ -661,14 +661,14 @@ ensure_paimon_vindex() {
fi
echo "Installing Python paimon-vindex dependency..."
- if python -m pip install 'paimon-vindex==0.4.0'; then
+ if python -m pip install 'paimon-vindex==0.5.0'; then
return 0
fi
echo -e "${YELLOW}Direct pip install failed; installing paimon-vindex into
a temporary target directory...${NC}"
local target_dir="${TMPDIR:-/tmp}/paimon-vindex-site"
rm -rf "$target_dir"
- if python -m pip install --target "$target_dir" 'paimon-vindex==0.4.0';
then
+ if python -m pip install --target "$target_dir" 'paimon-vindex==0.5.0';
then
export PYTHONPATH="$target_dir:${PYTHONPATH:-}"
return 0
fi
@@ -676,7 +676,7 @@ ensure_paimon_vindex() {
if python -c "import numpy" >/dev/null 2>&1; then
echo -e "${YELLOW}Dependency install failed but numpy is already
available; retrying paimon-vindex without dependencies...${NC}"
rm -rf "$target_dir"
- if python -m pip install --target "$target_dir" --no-deps
'paimon-vindex==0.4.0'; then
+ if python -m pip install --target "$target_dir" --no-deps
'paimon-vindex==0.5.0'; then
export PYTHONPATH="$target_dir:${PYTHONPATH:-}"
return 0
fi
diff --git a/paimon-python/pypaimon/globalindex/create_global_index.py
b/paimon-python/pypaimon/globalindex/create_global_index.py
index 0671ab09e1..09b8a568f5 100644
--- a/paimon-python/pypaimon/globalindex/create_global_index.py
+++ b/paimon-python/pypaimon/globalindex/create_global_index.py
@@ -123,6 +123,7 @@ class GlobalIndexBuilder:
self._index_type = index_type.lower().strip()
self._partition_filter = partition_filter
self._partitions = partitions
+ self._user_options = dict(options or {})
self._options = _merged_options(table, options)
self._core_options = CoreOptions(self._options)
@@ -464,8 +465,9 @@ class GlobalIndexBuilder:
index_path,
index_field.type,
self._index_type,
- self._options.to_map(),
+ self._table.options.options.to_map(),
index_field.name,
+ self._user_options,
)
if self._index_type == FULL_TEXT_IDENTIFIER:
return NativeFullTextIndexWriter(
diff --git
a/paimon-python/pypaimon/globalindex/vindex/vindex_vector_global_index_reader.py
b/paimon-python/pypaimon/globalindex/vindex/vindex_vector_global_index_reader.py
index c3fc51f61a..a8e8aee60c 100644
---
a/paimon-python/pypaimon/globalindex/vindex/vindex_vector_global_index_reader.py
+++
b/paimon-python/pypaimon/globalindex/vindex/vindex_vector_global_index_reader.py
@@ -205,7 +205,7 @@ class VindexVectorGlobalIndexReader(GlobalIndexReader):
except ImportError as e:
raise ImportError(
"paimon-vindex is required to read vindex vector indexes. "
- "Install paimon-vindex==0.4.0 or pypaimon[vindex].") from e
+ "Install paimon-vindex==0.5.0 or pypaimon[vindex].") from e
file_path = (self._io_meta.external_path
if self._io_meta.external_path
diff --git
a/paimon-python/pypaimon/globalindex/vindex/vindex_vector_index_writer.py
b/paimon-python/pypaimon/globalindex/vindex/vindex_vector_index_writer.py
index 96443cca34..eb15565c49 100644
--- a/paimon-python/pypaimon/globalindex/vindex/vindex_vector_index_writer.py
+++ b/paimon-python/pypaimon/globalindex/vindex/vindex_vector_index_writer.py
@@ -44,6 +44,7 @@ class VindexVectorIndexWriter:
index_type: str,
options: Mapping[str, object],
field_name: str,
+ user_options: Optional[Mapping[str, object]] = None,
):
self.file_name = (
"%s-%s-global-index-%s.index"
@@ -53,9 +54,9 @@ class VindexVectorIndexWriter:
self._index_path = index_path.rstrip('/')
self._index_type = index_type
self._native_options = native_options(
- data_type, options, index_type, field_name)
+ data_type, options, index_type, field_name, user_options)
self._train_sample_ratio = train_sample_ratio(
- options, index_type, field_name)
+ options, index_type, field_name, user_options)
self._dimension = int(self._native_options["dimension"])
self._row_count = 0
self._vector_count = 0
@@ -148,7 +149,7 @@ class VindexVectorIndexWriter:
except ImportError as e:
raise ImportError(
"paimon-vindex is required to build vindex vector indexes.
"
- "Install paimon-vindex==0.4.0 or pypaimon[vindex].") from e
+ "Install paimon-vindex==0.5.0 or pypaimon[vindex].") from e
self._close_temp_files()
self._file_io.check_or_mkdirs(self._index_path)
@@ -249,27 +250,19 @@ class VindexVectorIndexWriter:
def native_options(
data_type: DataType,
- options: Mapping[str, object],
+ table_options: Mapping[str, object],
index_type: str,
field_name: str,
+ user_options: Optional[Mapping[str, object]] = None,
) -> Dict[str, str]:
- result: Dict[str, str] = {}
- option_prefix = "%s." % index_type
- field_prefix = "fields.%s." % field_name
-
- for key, value in options.items():
- key = str(key)
- if key.startswith(option_prefix):
- native_key = _native_option_key(key[len(option_prefix):])
- if native_key is not None:
- result[native_key] = str(value)
+ if user_options is None:
+ user_options, table_options = table_options, {}
- for key, value in options.items():
- key = str(key)
- if key.startswith(field_prefix):
- native_key = _native_option_key(key[len(field_prefix):])
- if native_key is not None:
- result[native_key] = str(value)
+ result: Dict[str, str] = {}
+ _collect_native_options(
+ result, table_options, index_type, field_name, validate=False)
+ _collect_native_options(
+ result, user_options, index_type, field_name, validate=True)
result["index.type"] = index_type.replace('-', '_')
result["dimension"] = str(_dimension(data_type, result, index_type))
@@ -278,12 +271,19 @@ def native_options(
def train_sample_ratio(
- options: Mapping[str, object], index_type: str, field_name: str
+ table_options: Mapping[str, object],
+ index_type: str,
+ field_name: str,
+ user_options: Optional[Mapping[str, object]] = None,
) -> float:
+ sources = (table_options,) if user_options is None else (user_options,
table_options)
field_key = "fields.%s.train.sample-ratio" % field_name
index_key = "%s.train.sample-ratio" % index_type
- key = field_key if field_key in options else index_key
- if key not in options:
+ for options in sources:
+ key = field_key if field_key in options else index_key
+ if key in options:
+ break
+ else:
return 1.0
value = options[key]
@@ -322,6 +322,42 @@ def validate_vector_type(data_type: DataType) -> None:
% data_type)
+def _collect_native_options(
+ result, options, index_type, field_name, validate
+):
+ option_prefix = "%s." % index_type
+ field_prefix = "fields.%s." % field_name
+
+ for key, value in options.items():
+ key = str(key)
+ native_key = _native_option_key(key)
+ if key == native_key and _is_050_build_option(native_key):
+ _put_native_option(
+ result, key, key, value, index_type, validate)
+ for key, value in options.items():
+ key = str(key)
+ if key.startswith(option_prefix):
+ _put_native_option(
+ result,
+ key,
+ key[len(option_prefix):],
+ value,
+ index_type,
+ validate,
+ )
+ for key, value in options.items():
+ key = str(key)
+ if key.startswith(field_prefix):
+ _put_native_option(
+ result,
+ key,
+ key[len(field_prefix):],
+ value,
+ index_type,
+ validate,
+ )
+
+
def _native_option_key(option_key: str) -> Optional[str]:
if option_key in ("index.dimension", "dimension"):
return "dimension"
@@ -330,6 +366,10 @@ def _native_option_key(option_key: str) -> Optional[str]:
if option_key in (
"nlist",
"expected-vector-count",
+ "ivf.coarse-assignment",
+ "ivf.pq-encoding",
+ "ivf.train.max-points-per-centroid",
+ "pq.train.max-points-per-centroid",
"pq.m",
"pq.code-ratio",
"pq.bits",
@@ -366,6 +406,40 @@ def _native_option_key(option_key: str) -> Optional[str]:
return None
+def _put_native_option(
+ result, option_key, option_suffix, value, index_type, validate
+):
+ native_key = _native_option_key(option_suffix)
+ if native_key is None:
+ return
+ if (_is_050_build_option(native_key)
+ and not _is_allowed_050_build_option(native_key, index_type)):
+ if validate:
+ raise ValueError(
+ "Option '%s' is not supported for index type '%s'."
+ % (option_key, index_type)
+ )
+ return
+ result[native_key] = str(value)
+
+
+def _is_050_build_option(key):
+ return key in (
+ "ivf.coarse-assignment",
+ "ivf.pq-encoding",
+ "ivf.train.max-points-per-centroid",
+ "pq.train.max-points-per-centroid",
+ )
+
+
+def _is_allowed_050_build_option(key, index_type):
+ if key == "ivf.pq-encoding":
+ return index_type == "ivf-pq"
+ if key == "pq.train.max-points-per-centroid":
+ return index_type in ("ivf-pq", "diskann")
+ return index_type != "diskann"
+
+
def _dimension(
data_type: DataType, native_options_map: Mapping[str, str], index_type: str
) -> int:
diff --git a/paimon-python/pypaimon/tests/global_index_build_test.py
b/paimon-python/pypaimon/tests/global_index_build_test.py
index 9876d486fd..d7f117ae93 100644
--- a/paimon-python/pypaimon/tests/global_index_build_test.py
+++ b/paimon-python/pypaimon/tests/global_index_build_test.py
@@ -1086,14 +1086,14 @@ class GlobalIndexBuildTest(
}
rq_result = native_options(
- data_type, options, 'ivf-rq', 'embedding')
+ data_type, options, 'ivf-rq', 'embedding', {})
self.assertEqual('ivf_rq', rq_result['index.type'])
self.assertEqual('5', rq_result['rq.bits'])
self.assertEqual('96', rq_result['max-bytes-per-vector'])
self.assertEqual('inner_product', rq_result['metric'])
diskann_result = native_options(
- data_type, options, 'diskann', 'embedding')
+ data_type, options, 'diskann', 'embedding', {})
self.assertEqual('diskann', diskann_result['index.type'])
self.assertEqual(
'balanced', diskann_result['diskann.build-preset'])
@@ -1101,6 +1101,71 @@ class GlobalIndexBuildTest(
self.assertEqual(
'f16', diskann_result['diskann.raw-vector-encoding'])
+ def test_vindex_native_options_support_050_build_options(self):
+ data_type = ArrayType(True, AtomicType('FLOAT'))
+ native_names = {
+ 'ivf.coarse-assignment': 'auto',
+ 'ivf.pq-encoding': 'auto',
+ 'ivf.train.max-points-per-centroid': '32',
+ 'pq.train.max-points-per-centroid': '64',
+ }
+
+ result = native_options(
+ data_type, native_names, 'ivf-pq', 'embedding')
+ self.assertEqual('auto', result['ivf.coarse-assignment'])
+ self.assertEqual('auto', result['ivf.pq-encoding'])
+ self.assertEqual('32', result['ivf.train.max-points-per-centroid'])
+ self.assertEqual('64', result['pq.train.max-points-per-centroid'])
+
+ prefixed_names = {
+ 'ivf-pq.ivf.coarse-assignment': 'exact',
+ 'fields.embedding.ivf.pq-encoding': 'canonical',
+ 'ivf-pq.ivf.train.max-points-per-centroid': '16',
+ 'fields.embedding.pq.train.max-points-per-centroid': '48',
+ }
+ result = native_options(
+ data_type, prefixed_names, 'ivf-pq', 'embedding')
+ self.assertEqual('exact', result['ivf.coarse-assignment'])
+ self.assertEqual('canonical', result['ivf.pq-encoding'])
+ self.assertEqual('16', result['ivf.train.max-points-per-centroid'])
+ self.assertEqual('48', result['pq.train.max-points-per-centroid'])
+
+ result = native_options(
+ data_type,
+ {'diskann.pq.train.max-points-per-centroid': '24'},
+ 'diskann',
+ 'embedding',
+ )
+ self.assertEqual('24', result['pq.train.max-points-per-centroid'])
+
+ table_options = {'fields.embedding.ivf.pq-encoding': 'auto'}
+ result = native_options(
+ data_type,
+ table_options,
+ 'ivf-pq',
+ 'embedding',
+ {'ivf.pq-encoding': 'canonical'},
+ )
+ self.assertEqual('canonical', result['ivf.pq-encoding'])
+ result = native_options(
+ data_type, table_options, 'ivf-flat', 'embedding', {})
+ self.assertNotIn('ivf.pq-encoding', result)
+
+ with self.assertRaisesRegex(ValueError, 'ivf-flat.ivf.pq-encoding'):
+ native_options(
+ data_type,
+ {'ivf-flat.ivf.pq-encoding': 'canonical'},
+ 'ivf-flat',
+ 'embedding',
+ )
+ with self.assertRaisesRegex(ValueError,
'diskann.ivf.coarse-assignment'):
+ native_options(
+ data_type,
+ {'diskann.ivf.coarse-assignment': 'exact'},
+ 'diskann',
+ 'embedding',
+ )
+
def test_vindex_training_sample_ratio(self):
options = {
'ivf-rq.train.sample-ratio': '0.5',
@@ -1110,6 +1175,15 @@ class GlobalIndexBuildTest(
0.25, train_sample_ratio(options, 'ivf-rq', 'embedding'))
self.assertEqual(
0.5, train_sample_ratio(options, 'ivf-rq', 'other'))
+ self.assertEqual(
+ 0.5,
+ train_sample_ratio(
+ {'fields.embedding.train.sample-ratio': '0.75'},
+ 'ivf-rq',
+ 'embedding',
+ {'ivf-rq.train.sample-ratio': '0.5'},
+ ),
+ )
import numpy as np
vectors = np.arange(20, dtype=np.float32).reshape(10, 2)
diff --git a/paimon-python/setup.py b/paimon-python/setup.py
index 1af50be1e5..7d732167b8 100644
--- a/paimon-python/setup.py
+++ b/paimon-python/setup.py
@@ -294,7 +294,7 @@ setup(
'lumina-data>=0.1.0'
],
'vindex': [
- 'paimon-vindex==0.4.0; python_version>="3.9"',
+ 'paimon-vindex==0.5.0; python_version>="3.9"',
],
'full-text': [
'paimon-ftindex==0.1.0; python_version>="3.8"',
diff --git a/paimon-vector/pom.xml b/paimon-vector/pom.xml
index 4bedd7a8a4..058c0c934d 100644
--- a/paimon-vector/pom.xml
+++ b/paimon-vector/pom.xml
@@ -32,7 +32,7 @@ under the License.
<name>Paimon : Vector Index</name>
<properties>
-
<paimon-vector-index-java.version>0.4.0</paimon-vector-index-java.version>
+
<paimon-vector-index-java.version>0.5.0</paimon-vector-index-java.version>
</properties>
<dependencies>
diff --git
a/paimon-vector/src/main/java/org/apache/paimon/vector/index/NativeVectorGlobalIndexerFactory.java
b/paimon-vector/src/main/java/org/apache/paimon/vector/index/NativeVectorGlobalIndexerFactory.java
index 0f28479fa0..ad11f4bf16 100644
---
a/paimon-vector/src/main/java/org/apache/paimon/vector/index/NativeVectorGlobalIndexerFactory.java
+++
b/paimon-vector/src/main/java/org/apache/paimon/vector/index/NativeVectorGlobalIndexerFactory.java
@@ -48,32 +48,9 @@ public abstract class NativeVectorGlobalIndexerFactory
implements GlobalIndexerF
static Map<String, String> nativeOptions(
DataType fieldType, Options tableOptions, String identifier,
String fieldName) {
Map<String, String> nativeOptions = new LinkedHashMap<>();
- String optionPrefix = identifier + ".";
- String fieldPrefix = "fields." + fieldName + ".";
- Map<String, String> tableOptionsMap = tableOptions.toMap();
-
- // First collect index-type level options, e.g. <index-type>.xxx.
- for (Map.Entry<String, String> entry : tableOptionsMap.entrySet()) {
- String optionKey = entry.getKey();
- if (optionKey.startsWith(optionPrefix)) {
- String nativeKey =
nativeOptionKey(optionKey.substring(optionPrefix.length()));
- if (nativeKey != null) {
- nativeOptions.put(nativeKey, entry.getValue());
- }
- }
- }
-
- // Then collect field level options, e.g. fields.<field-name>.xxx,
which take precedence
- // over the index-type level options for this field.
- for (Map.Entry<String, String> entry : tableOptionsMap.entrySet()) {
- String optionKey = entry.getKey();
- if (optionKey.startsWith(fieldPrefix)) {
- String nativeKey =
nativeOptionKey(optionKey.substring(fieldPrefix.length()));
- if (nativeKey != null) {
- nativeOptions.put(nativeKey, entry.getValue());
- }
- }
- }
+ collectNativeOptions(nativeOptions, tableOptions.toMap(), identifier,
fieldName, false);
+ collectNativeOptions(
+ nativeOptions, tableOptions.dynamicOptions(), identifier,
fieldName, true);
nativeOptions.put("index.type", identifier.replace('-', '_'));
nativeOptions.put(
@@ -83,14 +60,19 @@ public abstract class NativeVectorGlobalIndexerFactory
implements GlobalIndexerF
}
static double trainSampleRatio(Options tableOptions, String identifier,
String fieldName) {
- Map<String, String> tableOptionsMap = tableOptions.toMap();
+ Map<String, String> source = tableOptions.dynamicOptions();
String key =
- resolveFieldOverriddenKey(
- tableOptionsMap, identifier, fieldName,
TRAIN_SAMPLE_RATIO_OPTION);
+ resolveFieldOverriddenKey(source, identifier, fieldName,
TRAIN_SAMPLE_RATIO_OPTION);
+ if (key == null) {
+ source = tableOptions.toMap();
+ key =
+ resolveFieldOverriddenKey(
+ source, identifier, fieldName,
TRAIN_SAMPLE_RATIO_OPTION);
+ }
if (key == null) {
return DEFAULT_TRAIN_SAMPLE_RATIO;
}
- String value = tableOptionsMap.get(key);
+ String value = source.get(key);
try {
double parsed = Double.parseDouble(value.trim());
@@ -138,6 +120,54 @@ public abstract class NativeVectorGlobalIndexerFactory
implements GlobalIndexerF
return null;
}
+ private static void collectNativeOptions(
+ Map<String, String> nativeOptions,
+ Map<String, String> options,
+ String identifier,
+ String fieldName,
+ boolean validate) {
+ String optionPrefix = identifier + ".";
+ String fieldPrefix = "fields." + fieldName + ".";
+
+ // Native names have the lowest precedence within each option source.
+ for (Map.Entry<String, String> entry : options.entrySet()) {
+ String nativeKey = nativeOptionKey(entry.getKey());
+ if (entry.getKey().equals(nativeKey) &&
is050BuildOption(nativeKey)) {
+ putNativeOption(
+ nativeOptions,
+ entry.getKey(),
+ entry.getKey(),
+ entry.getValue(),
+ identifier,
+ validate);
+ }
+ }
+ for (Map.Entry<String, String> entry : options.entrySet()) {
+ String optionKey = entry.getKey();
+ if (optionKey.startsWith(optionPrefix)) {
+ putNativeOption(
+ nativeOptions,
+ optionKey,
+ optionKey.substring(optionPrefix.length()),
+ entry.getValue(),
+ identifier,
+ validate);
+ }
+ }
+ for (Map.Entry<String, String> entry : options.entrySet()) {
+ String optionKey = entry.getKey();
+ if (optionKey.startsWith(fieldPrefix)) {
+ putNativeOption(
+ nativeOptions,
+ optionKey,
+ optionKey.substring(fieldPrefix.length()),
+ entry.getValue(),
+ identifier,
+ validate);
+ }
+ }
+ }
+
private static String nativeOptionKey(String optionKey) {
switch (optionKey) {
case "index.dimension":
@@ -148,6 +178,10 @@ public abstract class NativeVectorGlobalIndexerFactory
implements GlobalIndexerF
return "metric";
case "nlist":
case "expected-vector-count":
+ case "ivf.coarse-assignment":
+ case "ivf.pq-encoding":
+ case "ivf.train.max-points-per-centroid":
+ case "pq.train.max-points-per-centroid":
case "pq.m":
case "pq.code-ratio":
case "pq.bits":
@@ -191,6 +225,49 @@ public abstract class NativeVectorGlobalIndexerFactory
implements GlobalIndexerF
}
}
+ private static void putNativeOption(
+ Map<String, String> nativeOptions,
+ String optionKey,
+ String optionSuffix,
+ String value,
+ String identifier,
+ boolean validate) {
+ String nativeKey = nativeOptionKey(optionSuffix);
+ if (nativeKey == null) {
+ return;
+ }
+ if (is050BuildOption(nativeKey) && !isAllowed050BuildOption(nativeKey,
identifier)) {
+ if (validate) {
+ throw new IllegalArgumentException(
+ "Option '"
+ + optionKey
+ + "' is not supported for index type '"
+ + identifier
+ + "'.");
+ }
+ return;
+ }
+ nativeOptions.put(nativeKey, value);
+ }
+
+ private static boolean is050BuildOption(String key) {
+ return "ivf.coarse-assignment".equals(key)
+ || "ivf.pq-encoding".equals(key)
+ || "ivf.train.max-points-per-centroid".equals(key)
+ || "pq.train.max-points-per-centroid".equals(key);
+ }
+
+ private static boolean isAllowed050BuildOption(String key, String
identifier) {
+ if ("ivf.pq-encoding".equals(key)) {
+ return
IvfPqAlgorithmVectorGlobalIndexerFactory.IDENTIFIER.equals(identifier);
+ }
+ if ("pq.train.max-points-per-centroid".equals(key)) {
+ return
IvfPqAlgorithmVectorGlobalIndexerFactory.IDENTIFIER.equals(identifier)
+ ||
DiskAnnVectorGlobalIndexerFactory.IDENTIFIER.equals(identifier);
+ }
+ return
!DiskAnnVectorGlobalIndexerFactory.IDENTIFIER.equals(identifier);
+ }
+
private static int dimension(
DataType fieldType, Map<String, String> nativeOptions, String
identifier) {
if (fieldType instanceof VectorType) {
diff --git
a/paimon-vector/src/test/java/org/apache/paimon/vector/index/NativeVectorGlobalIndexerFactoryTest.java
b/paimon-vector/src/test/java/org/apache/paimon/vector/index/NativeVectorGlobalIndexerFactoryTest.java
index 6c30913491..c0e034b96f 100644
---
a/paimon-vector/src/test/java/org/apache/paimon/vector/index/NativeVectorGlobalIndexerFactoryTest.java
+++
b/paimon-vector/src/test/java/org/apache/paimon/vector/index/NativeVectorGlobalIndexerFactoryTest.java
@@ -26,6 +26,7 @@ import org.apache.paimon.types.VectorType;
import org.junit.jupiter.api.Test;
+import java.util.HashMap;
import java.util.Map;
import static org.assertj.core.api.Assertions.assertThat;
@@ -100,12 +101,13 @@ public class NativeVectorGlobalIndexerFactoryTest {
@Test
public void testNewVectorIndexOptions() {
- Options options = new Options();
- options.setString("ivf-rq.rq.bits", "5");
- options.setString("ivf-rq.max-bytes-per-vector", "96");
- options.setString("diskann.build-preset", "balanced");
- options.setString("diskann.pq.code-ratio", "0.0625");
- options.setString("diskann.raw-vector-encoding", "f16");
+ Map<String, String> tableOptions = new HashMap<>();
+ tableOptions.put("ivf-rq.rq.bits", "5");
+ tableOptions.put("ivf-rq.max-bytes-per-vector", "96");
+ tableOptions.put("diskann.build-preset", "balanced");
+ tableOptions.put("diskann.pq.code-ratio", "0.0625");
+ tableOptions.put("diskann.raw-vector-encoding", "f16");
+ Options options = new Options(tableOptions);
Map<String, String> rqOptions =
NativeVectorGlobalIndexerFactory.nativeOptions(
@@ -131,6 +133,114 @@ public class NativeVectorGlobalIndexerFactoryTest {
.containsEntry("diskann.raw-vector-encoding", "f16");
}
+ @Test
+ public void test050BuildOptions() {
+ Options nativeNames = new Options();
+ nativeNames.setString("ivf.coarse-assignment", "auto");
+ nativeNames.setString("ivf.pq-encoding", "auto");
+ nativeNames.setString("ivf.train.max-points-per-centroid", "32");
+ nativeNames.setString("pq.train.max-points-per-centroid", "64");
+
+ Map<String, String> nativeOptions =
+ NativeVectorGlobalIndexerFactory.nativeOptions(
+ new ArrayType(new FloatType()),
+ nativeNames,
+ IvfPqAlgorithmVectorGlobalIndexerFactory.IDENTIFIER,
+ "vec");
+ assertThat(nativeOptions)
+ .containsEntry("ivf.coarse-assignment", "auto")
+ .containsEntry("ivf.pq-encoding", "auto")
+ .containsEntry("ivf.train.max-points-per-centroid", "32")
+ .containsEntry("pq.train.max-points-per-centroid", "64");
+
+ Options prefixedNames = new Options();
+ prefixedNames.setString("ivf-pq.ivf.coarse-assignment", "exact");
+ prefixedNames.setString("fields.vec.ivf.pq-encoding", "canonical");
+ prefixedNames.setString("ivf-pq.ivf.train.max-points-per-centroid",
"16");
+ prefixedNames.setString("fields.vec.pq.train.max-points-per-centroid",
"48");
+ assertThat(
+ NativeVectorGlobalIndexerFactory.nativeOptions(
+ new ArrayType(new FloatType()),
+ prefixedNames,
+
IvfPqAlgorithmVectorGlobalIndexerFactory.IDENTIFIER,
+ "vec"))
+ .containsEntry("ivf.coarse-assignment", "exact")
+ .containsEntry("ivf.pq-encoding", "canonical")
+ .containsEntry("ivf.train.max-points-per-centroid", "16")
+ .containsEntry("pq.train.max-points-per-centroid", "48");
+
+ Options diskAnn = new Options();
+ diskAnn.setString("diskann.pq.train.max-points-per-centroid", "24");
+ assertThat(
+ NativeVectorGlobalIndexerFactory.nativeOptions(
+ new ArrayType(new FloatType()),
+ diskAnn,
+ DiskAnnVectorGlobalIndexerFactory.IDENTIFIER,
+ "vec"))
+ .containsEntry("pq.train.max-points-per-centroid", "24");
+
+ Map<String, String> tableOptions = new HashMap<>();
+ tableOptions.put("fields.vec.ivf.pq-encoding", "auto");
+ Map<String, String> userOptions = new HashMap<>();
+ userOptions.put("ivf.pq-encoding", "canonical");
+ assertThat(
+ NativeVectorGlobalIndexerFactory.nativeOptions(
+ new ArrayType(new FloatType()),
+ new Options(
+ new HashMap<>(), new
Options(tableOptions, userOptions)),
+
IvfPqAlgorithmVectorGlobalIndexerFactory.IDENTIFIER,
+ "vec"))
+ .containsEntry("ivf.pq-encoding", "canonical");
+ assertThat(
+ NativeVectorGlobalIndexerFactory.nativeOptions(
+ new ArrayType(new FloatType()),
+ new Options(tableOptions),
+ IvfFlatVectorGlobalIndexerFactory.IDENTIFIER,
+ "vec"))
+ .doesNotContainKey("ivf.pq-encoding");
+ }
+
+ @Test
+ public void testRejectsInapplicable050BuildOptions() {
+ Options mutableOptions = new Options();
+ mutableOptions.setString("ivf-flat.ivf.pq-encoding", "canonical");
+ assertThatThrownBy(
+ () ->
+ NativeVectorGlobalIndexerFactory.nativeOptions(
+ new ArrayType(new FloatType()),
+ mutableOptions,
+
IvfFlatVectorGlobalIndexerFactory.IDENTIFIER,
+ "vec"))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining("ivf-flat.ivf.pq-encoding");
+
+ Map<String, String> flatUserOptions = new HashMap<>();
+ flatUserOptions.put("ivf-flat.ivf.pq-encoding", "canonical");
+ Options flatOptions = new Options(new HashMap<>(), flatUserOptions);
+ assertThatThrownBy(
+ () ->
+ NativeVectorGlobalIndexerFactory.nativeOptions(
+ new ArrayType(new FloatType()),
+ flatOptions,
+
IvfFlatVectorGlobalIndexerFactory.IDENTIFIER,
+ "vec"))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining("ivf-flat.ivf.pq-encoding");
+
+ Map<String, String> diskAnnUserOptions = new HashMap<>();
+ diskAnnUserOptions.put("diskann.ivf.coarse-assignment", "exact");
+ Options diskAnnOptions = new Options(new HashMap<>(),
diskAnnUserOptions);
+ assertThatThrownBy(
+ () ->
+ NativeVectorGlobalIndexerFactory.nativeOptions(
+ new ArrayType(new FloatType()),
+ diskAnnOptions,
+
DiskAnnVectorGlobalIndexerFactory.IDENTIFIER,
+ "vec"))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining("diskann.ivf.coarse-assignment");
+ }
+
@Test
public void testNativeOptionsUsesVectorTypeDimension() {
Options options = new Options();
@@ -290,6 +400,17 @@ public class NativeVectorGlobalIndexerFactoryTest {
NativeVectorGlobalIndexerFactory.trainSampleRatio(
options,
IvfFlatVectorGlobalIndexerFactory.IDENTIFIER, "other"))
.isEqualTo(0.25);
+
+ Map<String, String> tableOptions = new HashMap<>();
+ tableOptions.put("fields.vec.train.sample-ratio", "0.75");
+ Map<String, String> userOptions = new HashMap<>();
+ userOptions.put("ivf-flat.train.sample-ratio", "0.5");
+ assertThat(
+ NativeVectorGlobalIndexerFactory.trainSampleRatio(
+ new Options(tableOptions, userOptions),
+ IvfFlatVectorGlobalIndexerFactory.IDENTIFIER,
+ "vec"))
+ .isEqualTo(0.5);
}
@Test
