This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new a910c615ac [global-index-eslib]use thin packaging and fix pom
bundle,fix query compatibility and vector metrics, and (#8618)
a910c615ac is described below
commit a910c615ac1e7ca44301043c938533180f3b9566
Author: CrownChu <[email protected]>
AuthorDate: Tue Jul 14 16:30:33 2026 +0800
[global-index-eslib]use thin packaging and fix pom bundle,fix query
compatibility and vector metrics, and (#8618)
This is a follow-up to #8000. It fixes persisted full-text query
compatibility and vector metric handling, and changes `paimon-eslib` to
a thin JAR so that ESLib and Lucene classes are no longer bundled into
the Apache artifact.
---
paimon-eslib/README.md | 5 +
paimon-eslib/pom.xml | 49 +------
.../eslib/index/ESIndexGlobalIndexReader.java | 68 ++++++++-
.../paimon/eslib/index/ESIndexGlobalIndexer.java | 55 ++++++-
.../apache/paimon/eslib/index/ESIndexOptions.java | 20 +++
.../index/ESIndexFullTextQueryParserTest.java | 26 ++++
.../eslib/index/ESIndexGlobalIndexE2ETest.java | 34 +++++
.../eslib/index/ESIndexVectorMetricTest.java | 162 +++++++++++++++++++++
8 files changed, 367 insertions(+), 52 deletions(-)
diff --git a/paimon-eslib/README.md b/paimon-eslib/README.md
index d64ef34a49..94a860bc3e 100644
--- a/paimon-eslib/README.md
+++ b/paimon-eslib/README.md
@@ -35,6 +35,11 @@ indexed according to their data type and per-field options.
> `es-index`. The module and its ESLib/Lucene dependencies require Java 11 or
> newer; the root Maven
> build intentionally skips `paimon-eslib` when it runs on JDK 8.
+`paimon-eslib` is distributed as a thin JAR and does not embed ESLib or Lucene
classes. Maven and
+Gradle resolve these dependencies transitively. When installing JARs manually,
place `eslib-core`,
+`eslib-simdvec`, and their Lucene 9.12 dependencies in the same runtime
classloader as
+`paimon-eslib`.
+
See the general [Global Index](../docs/docs/multimodal-table/global-index.mdx)
documentation for the
required Data Evolution table properties, coverage/freshness behavior, and
shared build options.
diff --git a/paimon-eslib/pom.xml b/paimon-eslib/pom.xml
index 8b9ea3ce80..a5fa8f5ee3 100644
--- a/paimon-eslib/pom.xml
+++ b/paimon-eslib/pom.xml
@@ -114,56 +114,11 @@ under the License.
<repositories>
<repository>
- <id>eslib-oss</id>
-
<url>https://paimon-es-public-bucket.oss-cn-beijing.aliyuncs.com/maven/</url>
+ <id>eslib-github</id>
+
<url>https://raw.githubusercontent.com/CrownChu/es-paimon-lib-releases/main/repository</url>
<releases><enabled>true</enabled></releases>
<snapshots><enabled>false</enabled></snapshots>
</repository>
</repositories>
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-shade-plugin</artifactId>
- <executions>
- <execution>
- <id>shade-eslib</id>
- <phase>package</phase>
- <goals>
- <goal>shade</goal>
- </goals>
- <configuration>
- <artifactSet>
- <includes>
-
<include>io.github.paimon.eslib:eslib-core</include>
-
<include>io.github.paimon.eslib:eslib-simdvec</include>
- <include>org.apache.lucene:*</include>
- </includes>
- </artifactSet>
- <relocations>
- <relocation>
- <pattern>org.apache.lucene</pattern>
-
<shadedPattern>org.apache.paimon.shade.lucene912</shadedPattern>
- </relocation>
- </relocations>
- <transformers>
- <transformer
implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
- </transformers>
- <filters>
- <filter>
- <artifact>*:*</artifact>
- <excludes>
- <exclude>module-info.class</exclude>
- <exclude>META-INF/MANIFEST.MF</exclude>
- <exclude>META-INF/versions/**</exclude>
- </excludes>
- </filter>
- </filters>
- </configuration>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
</project>
diff --git
a/paimon-eslib/src/main/java/org/apache/paimon/eslib/index/ESIndexGlobalIndexReader.java
b/paimon-eslib/src/main/java/org/apache/paimon/eslib/index/ESIndexGlobalIndexReader.java
index f95a269eb1..8e5bcd8ab7 100644
---
a/paimon-eslib/src/main/java/org/apache/paimon/eslib/index/ESIndexGlobalIndexReader.java
+++
b/paimon-eslib/src/main/java/org/apache/paimon/eslib/index/ESIndexGlobalIndexReader.java
@@ -350,6 +350,19 @@ public class ESIndexGlobalIndexReader implements
GlobalIndexReader {
return queryExecutor;
}
+ /** Returns the primary vector field's effective metric in Paimon's metric
vocabulary. */
+ String primaryVectorMetric() {
+ if (fields.isEmpty()) {
+ return null;
+ }
+ String physicalField = physicalField(fields.get(0).name());
+ FieldIndexConfig config =
+ physicalField == null ? null :
indexOptions.getConfig(physicalField);
+ return config != null && config.indexType() ==
FieldIndexConfig.IndexType.VECTOR
+ ? ESIndexOptions.toPaimonVectorMetric(config.metric())
+ : null;
+ }
+
int openStreamCount() {
synchronized (openProviders) {
return openProviders.size();
@@ -440,7 +453,7 @@ public class ESIndexGlobalIndexReader implements
GlobalIndexReader {
result.count,
DEFAULT_SAMPLE_LIMIT));
}
- return toScoredResult(result, topK);
+ return toVectorScoredResult(result, topK,
config.metric());
} catch (IOException e) {
throw new RuntimeException("Vector search failed", e);
}
@@ -632,9 +645,13 @@ public class ESIndexGlobalIndexReader implements
GlobalIndexReader {
}
if (q.has("match")) {
JsonNode m = requireObject(q.get("match"), "match query");
+ // Older SQL serialization includes a logical "column" routing
hint. Accept it for
+ // compatibility, but keep the already resolved physical field
authoritative so column
+ // renames continue to work.
requireOnlyFields(
m,
"match query",
+ "column",
"query",
"terms",
"operator",
@@ -650,7 +667,7 @@ public class ESIndexGlobalIndexReader implements
GlobalIndexReader {
requireObject(
q.has("match_phrase") ? q.get("match_phrase") :
q.get("phrase"),
"phrase query");
- requireOnlyFields(p, "phrase query", "query", "terms", "slop");
+ requireOnlyFields(p, "phrase query", "column", "query", "terms",
"slop");
int slop = p.has("slop") ? intValue(p.get("slop"), "slop") : 0;
return new FullTextQuerySpec.Phrase(field, queryText(p), slop);
} else if (q.has("boost")) {
@@ -967,6 +984,53 @@ public class ESIndexGlobalIndexReader implements
GlobalIndexReader {
return Optional.of(ScoredGlobalIndexResult.create(bitmap,
scoreMap::get));
}
+ private Optional<ScoredGlobalIndexResult> toVectorScoredResult(
+ SearchResult result, int limit, String metric) {
+ if (result == null || result.count == 0) {
+ return Optional.empty();
+ }
+
+ int count = Math.min(result.count, limit);
+ RoaringNavigableMap64 bitmap = new RoaringNavigableMap64();
+ Map<Long, Float> scoreMap = new HashMap<>(count);
+ for (int i = 0; i < count; i++) {
+ long id = result.ids[i];
+ bitmap.add(id);
+ scoreMap.put(id, toPaimonVectorScore(result.scores[i], metric));
+ }
+
+ return Optional.of(ScoredGlobalIndexResult.create(bitmap,
scoreMap::get));
+ }
+
+ /** Converts Lucene's metric-specific score into Paimon's exact
vector-search score. */
+ static float toPaimonVectorScore(float luceneScore, String metric) {
+ if (!Float.isFinite(luceneScore)) {
+ throw new IllegalArgumentException(
+ "Non-finite Lucene vector score for metric '" + metric +
"': " + luceneScore);
+ }
+ String normalized = metric == null ? "l2" :
metric.toLowerCase(Locale.ROOT);
+ switch (normalized) {
+ case "l2":
+ case "euclidean":
+ return luceneScore;
+ case "cosine":
+ case "dot_product":
+ case "dp":
+ return 2.0f * luceneScore - 1.0f;
+ case "inner_product":
+ case "mip":
+ case "maximum_inner_product":
+ if (luceneScore <= 0.0f) {
+ throw new IllegalArgumentException(
+ "Lucene maximum-inner-product score must be
positive; got: "
+ + luceneScore);
+ }
+ return luceneScore < 1.0f ? 1.0f - 1.0f / luceneScore :
luceneScore - 1.0f;
+ default:
+ throw new IllegalArgumentException("Unknown ESLib vector
metric: " + metric);
+ }
+ }
+
private void checkNotClosed() throws IOException {
if (closed) {
throw new IOException("Reader already closed");
diff --git
a/paimon-eslib/src/main/java/org/apache/paimon/eslib/index/ESIndexGlobalIndexer.java
b/paimon-eslib/src/main/java/org/apache/paimon/eslib/index/ESIndexGlobalIndexer.java
index 293e33a595..0167e24066 100644
---
a/paimon-eslib/src/main/java/org/apache/paimon/eslib/index/ESIndexGlobalIndexer.java
+++
b/paimon-eslib/src/main/java/org/apache/paimon/eslib/index/ESIndexGlobalIndexer.java
@@ -21,12 +21,14 @@ package org.apache.paimon.eslib.index;
import org.apache.paimon.globalindex.GlobalIndexIOMeta;
import org.apache.paimon.globalindex.GlobalIndexReader;
import org.apache.paimon.globalindex.GlobalIndexWriter;
-import org.apache.paimon.globalindex.GlobalIndexer;
+import org.apache.paimon.globalindex.VectorGlobalIndexer;
import org.apache.paimon.globalindex.io.GlobalIndexFileReader;
import org.apache.paimon.globalindex.io.GlobalIndexFileWriter;
import org.apache.paimon.options.Options;
import org.apache.paimon.types.DataField;
+import org.elasticsearch.eslib.api.model.FieldIndexConfig;
+
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
@@ -37,14 +39,17 @@ import java.util.concurrent.ExecutorService;
* ES multi-index global indexer using ESLib. Builds Lucene-based indexes
supporting vector,
* fulltext, and scalar fields.
*/
-public class ESIndexGlobalIndexer implements GlobalIndexer {
+public class ESIndexGlobalIndexer implements VectorGlobalIndexer {
private final List<DataField> fields;
private final ESIndexOptions indexOptions;
+ private final String configuredVectorMetric;
+ private volatile String readerVectorMetric;
public ESIndexGlobalIndexer(List<DataField> fields, Options options) {
this.fields = Collections.unmodifiableList(new ArrayList<>(fields));
this.indexOptions = new ESIndexOptions(this.fields, options);
+ this.configuredVectorMetric = primaryVectorMetric(this.fields,
this.indexOptions);
}
@Override
@@ -57,6 +62,50 @@ public class ESIndexGlobalIndexer implements GlobalIndexer {
GlobalIndexFileReader fileReader,
List<GlobalIndexIOMeta> files,
ExecutorService executor) {
- return new ESIndexGlobalIndexReader(fileReader, files, fields,
indexOptions, executor);
+ ESIndexGlobalIndexReader reader =
+ new ESIndexGlobalIndexReader(fileReader, files, fields,
indexOptions, executor);
+ try {
+ registerReaderVectorMetric(reader.primaryVectorMetric());
+ return reader;
+ } catch (RuntimeException e) {
+ try {
+ reader.close();
+ } catch (IOException closeFailure) {
+ e.addSuppressed(closeFailure);
+ }
+ throw e;
+ }
+ }
+
+ @Override
+ public String metric() {
+ String metric = readerVectorMetric;
+ return metric == null ? configuredVectorMetric : metric;
+ }
+
+ private synchronized void registerReaderVectorMetric(String metric) {
+ if (metric == null) {
+ return;
+ }
+ if (readerVectorMetric == null) {
+ readerVectorMetric = metric;
+ } else if (!readerVectorMetric.equals(metric)) {
+ throw new IllegalArgumentException(
+ "Cannot combine es-index shards with different vector
metrics: "
+ + readerVectorMetric
+ + " and "
+ + metric
+ + ".");
+ }
+ }
+
+ private static String primaryVectorMetric(List<DataField> fields,
ESIndexOptions indexOptions) {
+ if (fields.isEmpty()) {
+ return null;
+ }
+ FieldIndexConfig config = indexOptions.getConfig(fields.get(0).name());
+ return config != null && config.indexType() ==
FieldIndexConfig.IndexType.VECTOR
+ ? ESIndexOptions.toPaimonVectorMetric(config.metric())
+ : null;
}
}
diff --git
a/paimon-eslib/src/main/java/org/apache/paimon/eslib/index/ESIndexOptions.java
b/paimon-eslib/src/main/java/org/apache/paimon/eslib/index/ESIndexOptions.java
index 530e5fbe00..d1936ec21c 100644
---
a/paimon-eslib/src/main/java/org/apache/paimon/eslib/index/ESIndexOptions.java
+++
b/paimon-eslib/src/main/java/org/apache/paimon/eslib/index/ESIndexOptions.java
@@ -159,6 +159,26 @@ public class ESIndexOptions {
return fieldConfigs.get(fieldName);
}
+ /** Maps an ESLib/Lucene vector metric name to Paimon's exact-search
metric vocabulary. */
+ static String toPaimonVectorMetric(String metric) {
+ String normalized = metric == null ? "l2" :
metric.toLowerCase(Locale.ROOT);
+ switch (normalized) {
+ case "l2":
+ case "euclidean":
+ return "l2";
+ case "cosine":
+ return "cosine";
+ case "dot_product":
+ case "dp":
+ case "inner_product":
+ case "mip":
+ case "maximum_inner_product":
+ return "inner_product";
+ default:
+ throw new IllegalArgumentException("Unknown ESLib vector
metric: " + metric);
+ }
+ }
+
/** Returns the keyword multi-field sub-field name for {@code fieldName}
if one exists. */
public String keywordSubField(String fieldName) {
String subField = fieldName + KEYWORD_SUBFIELD_SUFFIX;
diff --git
a/paimon-eslib/src/test/java/org/apache/paimon/eslib/index/ESIndexFullTextQueryParserTest.java
b/paimon-eslib/src/test/java/org/apache/paimon/eslib/index/ESIndexFullTextQueryParserTest.java
index adb86557bc..73657470f3 100644
---
a/paimon-eslib/src/test/java/org/apache/paimon/eslib/index/ESIndexFullTextQueryParserTest.java
+++
b/paimon-eslib/src/test/java/org/apache/paimon/eslib/index/ESIndexFullTextQueryParserTest.java
@@ -27,6 +27,32 @@ import static
org.assertj.core.api.Assertions.assertThatThrownBy;
class ESIndexFullTextQueryParserTest {
+ @Test
+ void acceptsPersistedColumnRoutingHints() {
+ FullTextQuerySpec.Match match =
+ (FullTextQuerySpec.Match)
+ ESIndexGlobalIndexReader.parseSpec(
+ "physical_text",
+
"{\"match\":{\"column\":\"logical_text\",\"terms\":\"paimon\"}}");
+ assertThat(match.field()).isEqualTo("physical_text");
+ assertThat(match.text()).isEqualTo("paimon");
+
+ FullTextQuerySpec.Phrase phrase =
+ (FullTextQuerySpec.Phrase)
+ ESIndexGlobalIndexReader.parseSpec(
+ "physical_text",
+
"{\"match_phrase\":{\"column\":\"logical_text\",\"terms\":\"apache
paimon\",\"slop\":1}}");
+ assertThat(phrase.field()).isEqualTo("physical_text");
+ assertThat(phrase.text()).isEqualTo("apache paimon");
+ assertThat(phrase.slop()).isEqualTo(1);
+
+ assertThat(
+ ESIndexGlobalIndexReader.parseSpec(
+ "physical_text",
+
"{\"boolean\":{\"queries\":[[\"Must\",{\"match\":{\"column\":\"logical_text\",\"terms\":\"paimon\"}}],[\"Should\",{\"phrase\":{\"column\":\"logical_text\",\"terms\":\"apache
paimon\"}}]]}}"))
+ .isNotNull();
+ }
+
@Test
void parsesCaseInsensitiveBooleanOperatorsAndRejectsTypos() {
FullTextQuerySpec.Match and =
diff --git
a/paimon-eslib/src/test/java/org/apache/paimon/eslib/index/ESIndexGlobalIndexE2ETest.java
b/paimon-eslib/src/test/java/org/apache/paimon/eslib/index/ESIndexGlobalIndexE2ETest.java
index 27ec952689..464d1ac8a4 100644
---
a/paimon-eslib/src/test/java/org/apache/paimon/eslib/index/ESIndexGlobalIndexE2ETest.java
+++
b/paimon-eslib/src/test/java/org/apache/paimon/eslib/index/ESIndexGlobalIndexE2ETest.java
@@ -1636,6 +1636,40 @@ class ESIndexGlobalIndexE2ETest {
entry.meta());
}
+ @Test
+ void vectorSearchReturnsPaimonCosineScoreScale(@TempDir java.nio.file.Path
tmp)
+ throws Exception {
+ List<DataField> fields =
+ List.of(new DataField(0, "embedding",
DataTypes.ARRAY(DataTypes.FLOAT())));
+ Map<String, String> optionMap = new HashMap<>();
+ optionMap.put("global-index.es-index.fields.embedding.dimension", "2");
+ optionMap.put("global-index.es-index.fields.embedding.metric",
"cosine");
+ ESIndexOptions options = new ESIndexOptions(fields,
Options.fromMap(optionMap));
+
+ java.nio.file.Path archiveDir = tmp.resolve("cosine-scores");
+ Files.createDirectories(archiveDir);
+ ESIndexGlobalIndexWriter writer =
+ new ESIndexGlobalIndexWriter(new LocalDirWriter(archiveDir),
fields, options);
+ writer.write(new float[] {1.0f, 0.0f}, 0L);
+ writer.write(new float[] {0.0f, 1.0f}, 1L);
+ ResultEntry entry = writer.finish().get(0);
+
+ ESIndexGlobalIndexReader reader =
+ new ESIndexGlobalIndexReader(
+ new LocalFileReader(), List.of(ioMeta(archiveDir,
entry)), fields, options);
+ try {
+ ScoredGlobalIndexResult result =
+ reader.visitVectorSearch(
+ new VectorSearch(new float[] {1.0f, 0.0f},
2, "embedding"))
+ .join()
+ .orElseThrow(AssertionError::new);
+ assertEquals(1.0f, result.scoreGetter().score(0L), 0.000001f);
+ assertEquals(0.0f, result.scoreGetter().score(1L), 0.000001f);
+ } finally {
+ reader.close();
+ }
+ }
+
/**
* Verifies the DiskBBQ vector codec write path through {@link
ESIndexGlobalIndexWriter}.
*
diff --git
a/paimon-eslib/src/test/java/org/apache/paimon/eslib/index/ESIndexVectorMetricTest.java
b/paimon-eslib/src/test/java/org/apache/paimon/eslib/index/ESIndexVectorMetricTest.java
new file mode 100644
index 0000000000..fceab62411
--- /dev/null
+++
b/paimon-eslib/src/test/java/org/apache/paimon/eslib/index/ESIndexVectorMetricTest.java
@@ -0,0 +1,162 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.eslib.index;
+
+import org.apache.paimon.globalindex.GlobalIndexIOMeta;
+import org.apache.paimon.globalindex.GlobalIndexReader;
+import org.apache.paimon.globalindex.GlobalIndexer;
+import org.apache.paimon.globalindex.VectorGlobalIndexer;
+import org.apache.paimon.options.Options;
+import org.apache.paimon.types.DataField;
+import org.apache.paimon.types.DataTypes;
+
+import org.elasticsearch.eslib.api.model.FieldIndexConfig;
+import org.elasticsearch.eslib.api.model.VectorAlgorithm;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+class ESIndexVectorMetricTest {
+
+ private static final List<DataField> VECTOR_FIELDS =
+ List.of(new DataField(0, "embedding", DataTypes.VECTOR(2,
DataTypes.FLOAT())));
+
+ @Test
+ void exposesCanonicalPaimonMetric() {
+ GlobalIndexer indexer = indexer("dp");
+ assertThat(indexer).isInstanceOf(VectorGlobalIndexer.class);
+ assertThat(((VectorGlobalIndexer)
indexer).metric()).isEqualTo("inner_product");
+
+ assertThat(indexer("euclidean").metric()).isEqualTo("l2");
+ assertThat(indexer("cosine").metric()).isEqualTo("cosine");
+ assertThat(indexer("dot_product").metric()).isEqualTo("inner_product");
+
assertThat(indexer("inner_product").metric()).isEqualTo("inner_product");
+ assertThat(indexer("mip").metric()).isEqualTo("inner_product");
+
assertThat(indexer("maximum_inner_product").metric()).isEqualTo("inner_product");
+ }
+
+ @Test
+ void persistedMetricOverridesCurrentTableConfiguration(@TempDir Path
tempDir) throws Exception {
+ ESIndexGlobalIndexer indexer = indexer("cosine");
+ assertThat(indexer.metric()).isEqualTo("cosine");
+
+ GlobalIndexReader reader =
+ indexer.createReader(
+ meta -> null,
+ List.of(vectorFile(tempDir, "persisted-l2",
"euclidean")),
+ null);
+ try {
+ assertThat(((ESIndexGlobalIndexReader)
reader).primaryVectorMetric()).isEqualTo("l2");
+ assertThat(indexer.metric()).isEqualTo("l2");
+ } finally {
+ reader.close();
+ }
+ }
+
+ @Test
+ void rejectsShardsWithDifferentPersistedMetrics(@TempDir Path tempDir)
throws Exception {
+ ESIndexGlobalIndexer indexer = indexer("cosine");
+ GlobalIndexReader first =
+ indexer.createReader(
+ meta -> null, List.of(vectorFile(tempDir, "first",
"euclidean")), null);
+ try {
+ assertThatThrownBy(
+ () ->
+ indexer.createReader(
+ meta -> null,
+ List.of(vectorFile(tempDir,
"second", "cosine")),
+ null))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining("different vector metrics")
+ .hasMessageContaining("l2")
+ .hasMessageContaining("cosine");
+ } finally {
+ first.close();
+ }
+ }
+
+ @Test
+ void convertsLuceneScoresToPaimonScoreScale() {
+ assertThat(ESIndexGlobalIndexReader.toPaimonVectorScore(0.25f,
"l2")).isEqualTo(0.25f);
+ assertThat(ESIndexGlobalIndexReader.toPaimonVectorScore(0.25f,
"euclidean"))
+ .isEqualTo(0.25f);
+
+ assertThat(ESIndexGlobalIndexReader.toPaimonVectorScore(0.75f,
"cosine")).isEqualTo(0.5f);
+ assertThat(ESIndexGlobalIndexReader.toPaimonVectorScore(0.75f,
"dot_product"))
+ .isEqualTo(0.5f);
+ assertThat(ESIndexGlobalIndexReader.toPaimonVectorScore(0.25f,
"dp")).isEqualTo(-0.5f);
+
+ assertThat(ESIndexGlobalIndexReader.toPaimonVectorScore(3.0f,
"inner_product"))
+ .isEqualTo(2.0f);
+ assertThat(ESIndexGlobalIndexReader.toPaimonVectorScore(0.5f,
"mip")).isEqualTo(-1.0f);
+ assertThat(ESIndexGlobalIndexReader.toPaimonVectorScore(0.25f,
"maximum_inner_product"))
+ .isEqualTo(-3.0f);
+
+ assertThatThrownBy(() ->
ESIndexGlobalIndexReader.toPaimonVectorScore(Float.NaN, "cosine"))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining("Non-finite");
+ assertThatThrownBy(() ->
ESIndexGlobalIndexReader.toPaimonVectorScore(0.0f, "mip"))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining("must be positive");
+ }
+
+ private static ESIndexGlobalIndexer indexer(String metric) {
+ Map<String, String> options = new LinkedHashMap<>();
+ options.put("global-index.es-index.fields.embedding.metric", metric);
+ return new ESIndexGlobalIndexer(VECTOR_FIELDS,
Options.fromMap(options));
+ }
+
+ private static GlobalIndexIOMeta vectorFile(Path tempDir, String name,
String metric)
+ throws Exception {
+ Path segment = tempDir.resolve("segments_" + name);
+ Files.write(segment, new byte[] {1});
+
+ Map<String, FieldIndexConfig> configs = new LinkedHashMap<>();
+ configs.put(
+ "embedding",
+ FieldIndexConfig.builder("embedding",
FieldIndexConfig.IndexType.VECTOR)
+ .algorithm(VectorAlgorithm.HNSW)
+ .dimension(2)
+ .metric(metric)
+ .build());
+ byte[] metadata =
+ ESIndexFileMeta.write(
+ new java.io.File[] {segment.toFile()},
+ List.of("embedding"),
+
List.of(VECTOR_FIELDS.get(0).type().copy(true).asSQLString()),
+ configs);
+
+ long archiveSize = 0L;
+ for (long[] range :
ESIndexFileMeta.read(metadata).fileOffsets().values()) {
+ archiveSize = Math.max(archiveSize, range[0] + range[1]);
+ }
+ return new GlobalIndexIOMeta(
+ new org.apache.paimon.fs.Path(tempDir.resolve(name +
".index").toString()),
+ archiveSize,
+ metadata);
+ }
+}
