This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new d82e90d6f7 [global-index-eslib] Isolate dependencies and support int8
HNSW (#8702)
d82e90d6f7 is described below
commit d82e90d6f768e4701132c7308a18247b45a15f52
Author: CrownChu <[email protected]>
AuthorDate: Sat Jul 18 12:05:51 2026 +0800
[global-index-eslib] Isolate dependencies and support int8 HNSW (#8702)
---
.github/workflows/utcase-eslib.yml | 73 ++++++++++++++++++++++
paimon-eslib/README.md | 26 +++++---
paimon-eslib/pom.xml | 12 ++--
.../apache/paimon/eslib/index/ESIndexOptions.java | 15 +++--
.../paimon/eslib/index/ESIndexOptionsTest.java | 20 ++++++
pom.xml | 11 +---
6 files changed, 130 insertions(+), 27 deletions(-)
diff --git a/.github/workflows/utcase-eslib.yml
b/.github/workflows/utcase-eslib.yml
new file mode 100644
index 0000000000..a5dc7d9d20
--- /dev/null
+++ b/.github/workflows/utcase-eslib.yml
@@ -0,0 +1,73 @@
+################################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+name: UTCase ESLib on JDK 11
+
+on:
+ push:
+ paths:
+ - 'paimon-api/**'
+ - 'paimon-common/**'
+ - 'paimon-eslib/**'
+ - 'paimon-test-utils/**'
+ - 'pom.xml'
+ - '.github/workflows/utcase-eslib.yml'
+ pull_request:
+ paths:
+ - 'paimon-api/**'
+ - 'paimon-common/**'
+ - 'paimon-eslib/**'
+ - 'paimon-test-utils/**'
+ - 'pom.xml'
+ - '.github/workflows/utcase-eslib.yml'
+
+permissions:
+ contents: read
+
+env:
+ JDK_VERSION: 11
+ MAVEN_OPTS: -Dmaven.wagon.httpconnectionManager.ttlSeconds=30
-Dmaven.wagon.http.retryHandler.requestSentEnabled=true
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.event_name }}-${{
github.event.number || github.run_id }}
+ cancel-in-progress: true
+
+jobs:
+ eslib_test:
+ runs-on: ubuntu-latest
+ timeout-minutes: 30
+
+ steps:
+ - name: Checkout code
+ uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6
+ with:
+ persist-credentials: false
+
+ - name: Set up JDK ${{ env.JDK_VERSION }}
+ uses: actions/setup-java@03ad4de0992f5dab5e18fcb136590ce7c4a0ac95 # v5
+ with:
+ java-version: ${{ env.JDK_VERSION }}
+ distribution: 'temurin'
+
+ - name: Build ESLib dependencies
+ run: mvn -T 2C -B -ntp clean install -DskipTests -Ppaimon-eslib -pl
paimon-eslib -am
+
+ - name: Test ESLib
+ run: mvn -B -ntp verify -Ppaimon-eslib -pl paimon-eslib
+ env:
+ MAVEN_OPTS: -Xmx4096m
diff --git a/paimon-eslib/README.md b/paimon-eslib/README.md
index 94a860bc3e..95b4e93577 100644
--- a/paimon-eslib/README.md
+++ b/paimon-eslib/README.md
@@ -32,13 +32,23 @@ indexed according to their data type and per-field options.
> **Plugin required.** `es-index` is provided by this `paimon-eslib` module.
> Make sure `paimon-eslib`
> is on the classpath of your Spark or Flink job (and of any reader) before
> building or querying an
-> `es-index`. The module and its ESLib/Lucene dependencies require Java 11 or
newer; the root Maven
-> build intentionally skips `paimon-eslib` when it runs on JDK 8.
+> `es-index`. The module and its ESLib/Lucene dependencies require Java 11 or
newer. It is excluded
+> from the default Maven reactor; build it explicitly with `-Ppaimon-eslib` on
JDK 11 or newer.
`paimon-eslib` is distributed as a thin JAR and does not embed ESLib or Lucene
classes. Maven and
-Gradle resolve these dependencies transitively. When installing JARs manually,
place `eslib-core`,
-`eslib-simdvec`, and their Lucene 9.12 dependencies in the same runtime
classloader as
-`paimon-eslib`.
+Gradle resolve these dependencies transitively. When installing JARs manually,
place
+`eslib-core-lucene9` and `eslib-simdvec-lucene9` 1.0.7, together with their
Lucene 9.12
+dependencies, in the same runtime classloader as `paimon-eslib`.
+
+## ESLib source and artifacts
+
+- ESLib source code:
[CrownChu/es-paimon-lib](https://github.com/CrownChu/es-paimon-lib)
+- Published ESLib JARs:
+
[CrownChu/es-paimon-lib-releases](https://github.com/CrownChu/es-paimon-lib-releases)
+
+The ESLib source and binary repositories are maintained independently of the
Apache Software
+Foundation. The exact ESLib artifact versions used by this module are declared
in
+[`pom.xml`](pom.xml).
See the general [Global Index](../docs/docs/multimodal-table/global-index.mdx)
documentation for the
required Data Evolution table properties, coverage/freshness behavior, and
shared build options.
@@ -138,11 +148,11 @@ column.
| Key | Applies to | Default | Description |
|---|---|---|---|
| `type` | any column | inferred from data type | Force the sub-index type:
`vector`, `fulltext`, `keyword`, `date`. |
-| `algorithm` | vector | `hnsw` | Vector algorithm: `hnsw` or `diskbbq`.
`native` is not supported by paimon-eslib. |
+| `algorithm` | vector | `hnsw` | Vector algorithm: `hnsw`, `int8_hnsw`, or
`diskbbq`. `native` is not supported by paimon-eslib. |
| `dimension` | vector | inferred for `VECTOR<FLOAT>` | Vector dimension
(1–4096). Required for `ARRAY<FLOAT>`; taken from the type for `VECTOR<FLOAT>`.
|
| `metric` | vector | `cosine` | Distance metric (for example `cosine`, `l2`,
`dot_product`). Use the same metric at build and query time. |
-| `m` | vector (`hnsw`) | engine default | HNSW graph out-degree; valid range:
1–512. |
-| `ef_construction` | vector (`hnsw`) | engine default | HNSW construction
search width; valid range: 1–3200. |
+| `m` | vector (`hnsw`, `int8_hnsw`) | engine default | HNSW graph out-degree;
valid range: 1–512. |
+| `ef_construction` | vector (`hnsw`, `int8_hnsw`) | engine default | HNSW
construction search width; valid range: 1–3200. |
| `vectors_per_cluster` | vector (`diskbbq`) | engine default | Target vectors
per cluster for DiskBBQ; valid range: 64–65536. |
| `centroids_per_parent_cluster` | vector (`diskbbq`) | engine default |
Number of child centroids per parent cluster for DiskBBQ; valid range: 2–384. |
| `analyzer` | string | `standard` | Text analyzer used by the full-text
primary field or `.fulltext` sub-field: `standard`, `whitespace`, `simple`, or
`keyword`. |
diff --git a/paimon-eslib/pom.xml b/paimon-eslib/pom.xml
index a5fa8f5ee3..a6ada97e5f 100644
--- a/paimon-eslib/pom.xml
+++ b/paimon-eslib/pom.xml
@@ -35,6 +35,7 @@ under the License.
<!-- eslib-api / eslib-core are compiled for Java 11 (Lucene 9.12);
compile this module
at Java 11 instead of the project default (1.8) so their class
files are readable. -->
<target.java.version>11</target.java.version>
+ <eslib.version>1.0.7</eslib.version>
</properties>
<dependencies>
@@ -49,9 +50,8 @@ under the License.
into eslib-core), so no separate eslib-api artifact is needed. -->
<dependency>
<groupId>io.github.paimon.eslib</groupId>
- <artifactId>eslib-core</artifactId>
- <version>1.0.6</version>
- <classifier>lucene9</classifier>
+ <artifactId>eslib-core-lucene9</artifactId>
+ <version>${eslib.version}</version>
<!-- Paimon supplies its own ArchiveDataProvider over FileIO; the
standalone
OSSRangeIndexInput and Aliyun SDK are not used in this
module. -->
<exclusions>
@@ -63,8 +63,8 @@ under the License.
</dependency>
<dependency>
<groupId>io.github.paimon.eslib</groupId>
- <artifactId>eslib-simdvec</artifactId>
- <version>1.0.6</version>
+ <artifactId>eslib-simdvec-lucene9</artifactId>
+ <version>${eslib.version}</version>
</dependency>
<!-- eslib-native (Havenask JNI) intentionally omitted: it is a JDK 21
module (Phase 3,
not implemented) and is not referenced by paimon-eslib. Re-add
when NATIVE algorithm
@@ -115,7 +115,7 @@ under the License.
<repositories>
<repository>
<id>eslib-github</id>
-
<url>https://raw.githubusercontent.com/CrownChu/es-paimon-lib-releases/main/repository</url>
+
<url>https://raw.githubusercontent.com/CrownChu/es-paimon-lib-releases/eslib-${eslib.version}/repository</url>
<releases><enabled>true</enabled></releases>
<snapshots><enabled>false</enabled></snapshots>
</repository>
diff --git
a/paimon-eslib/src/main/java/org/apache/paimon/eslib/index/ESIndexOptions.java
b/paimon-eslib/src/main/java/org/apache/paimon/eslib/index/ESIndexOptions.java
index d1936ec21c..60b8a704f7 100644
---
a/paimon-eslib/src/main/java/org/apache/paimon/eslib/index/ESIndexOptions.java
+++
b/paimon-eslib/src/main/java/org/apache/paimon/eslib/index/ESIndexOptions.java
@@ -307,7 +307,7 @@ public class ESIndexOptions {
if (vectorAlgorithm == VectorAlgorithm.NATIVE) {
throw new IllegalArgumentException(
"Vector algorithm 'native' is not supported by
paimon-eslib; "
- + "use 'hnsw' or 'diskbbq'");
+ + "use 'hnsw', 'int8_hnsw', or 'diskbbq'");
}
String dimStr = resolve(options, fieldName, "dimension", null);
@@ -352,15 +352,16 @@ public class ESIndexOptions {
// table can contain HNSW and DiskBBQ fields at the same time, so a
global HNSW parameter
// must not make a DiskBBQ field invalid (and vice versa). Field-level
parameters remain
// strict because they unambiguously target this field.
+ boolean hnswAlgorithm = isHnswAlgorithm(vectorAlgorithm);
String mStr =
- vectorAlgorithm == VectorAlgorithm.HNSW
+ hnswAlgorithm
? resolve(options, fieldName, "m", null)
: resolveField(options, fieldName, "m");
String efStr =
- vectorAlgorithm == VectorAlgorithm.HNSW
+ hnswAlgorithm
? resolve(options, fieldName, "ef_construction", null)
: resolveField(options, fieldName, "ef_construction");
- if (vectorAlgorithm == VectorAlgorithm.HNSW) {
+ if (hnswAlgorithm) {
putBoundedPositiveIntegerParameter(
params, fieldName, "m", mStr,
Lucene99HnswVectorsFormat.MAXIMUM_MAX_CONN);
putBoundedPositiveIntegerParameter(
@@ -375,7 +376,7 @@ public class ESIndexOptions {
+ fieldName
+ "' configures m/ef_construction but algorithm is
"
+ algorithm
- + "; these parameters require algorithm=hnsw.");
+ + "; these parameters require algorithm=hnsw or
int8_hnsw.");
}
String vpcStr =
vectorAlgorithm == VectorAlgorithm.DISKBBQ
@@ -421,6 +422,10 @@ public class ESIndexOptions {
.build();
}
+ private static boolean isHnswAlgorithm(VectorAlgorithm algorithm) {
+ return algorithm == VectorAlgorithm.HNSW || algorithm ==
VectorAlgorithm.INT8_HNSW;
+ }
+
private static BuiltinAnalyzer parseAnalyzer(String fieldName, String
analyzerName) {
BuiltinAnalyzer analyzer = BuiltinAnalyzer.fromName(analyzerName);
if (analyzer == BuiltinAnalyzer.IK_SMART || analyzer ==
BuiltinAnalyzer.IK_MAX_WORD) {
diff --git
a/paimon-eslib/src/test/java/org/apache/paimon/eslib/index/ESIndexOptionsTest.java
b/paimon-eslib/src/test/java/org/apache/paimon/eslib/index/ESIndexOptionsTest.java
index eb1690bb50..637e9f9872 100644
---
a/paimon-eslib/src/test/java/org/apache/paimon/eslib/index/ESIndexOptionsTest.java
+++
b/paimon-eslib/src/test/java/org/apache/paimon/eslib/index/ESIndexOptionsTest.java
@@ -25,6 +25,7 @@ import org.apache.paimon.types.DataTypes;
import org.apache.lucene.codecs.KnnVectorsFormat;
import org.apache.lucene.codecs.perfield.PerFieldKnnVectorsFormat;
import org.elasticsearch.eslib.adapter.PaimonHnswVectorsFormat;
+import org.elasticsearch.eslib.adapter.PaimonInt8HnswVectorsFormat;
import org.elasticsearch.eslib.adapter.lucene9.PaimonLucene9Codec;
import org.elasticsearch.eslib.api.model.FieldIndexConfig;
import org.elasticsearch.eslib.api.model.ScalarFieldType;
@@ -108,6 +109,25 @@ class ESIndexOptionsTest {
assertThat(format.toString()).contains("maxConn=30").contains("beamWidth=360");
}
+ @Test
+ void int8HnswConstructionParametersReachLuceneCodec() {
+ Map<String, String> m = new HashMap<>();
+ m.put("global-index.es-index.fields.embedding.algorithm", "int8_hnsw");
+ m.put("global-index.es-index.fields.embedding.dimension", "128");
+ m.put("global-index.es-index.fields.embedding.m", "24");
+ m.put("global-index.es-index.fields.embedding.ef_construction", "160");
+
+ ESIndexOptions options = new ESIndexOptions(FIELDS,
Options.fromMap(m));
+ FieldIndexConfig config = options.getConfig("embedding");
+ PaimonLucene9Codec codec = new
PaimonLucene9Codec(options.getFieldConfigs());
+ PerFieldKnnVectorsFormat perField = (PerFieldKnnVectorsFormat)
codec.knnVectorsFormat();
+ KnnVectorsFormat format =
perField.getKnnVectorsFormatForField("embedding");
+
+ assertThat(config.algorithm()).isEqualTo(VectorAlgorithm.INT8_HNSW);
+ assertThat(format).isInstanceOf(PaimonInt8HnswVectorsFormat.class);
+
assertThat(format.toString()).contains("maxConn=24").contains("beamWidth=160");
+ }
+
@Test
void invalidHnswConstructionParametersAreRejected() {
Map<String, String> nonPositive = new HashMap<>();
diff --git a/pom.xml b/pom.xml
index 03b4c4ee2f..9465363a39 100644
--- a/pom.xml
+++ b/pom.xml
@@ -521,18 +521,13 @@ under the License.
</profile>
<profile>
- <!-- paimon-eslib depends on Java-11 eslib-api/eslib-core (Lucene
9.12), so it must be
- built with JDK 11+. The [11,) range activates on JDK
11/17/21; a bare
- <jdk>11</jdk> is a prefix match that only fires on JDK 11,
silently skipping the
- module (no compile, no test) on 17/21. The default JDK-8
build still skips it, so
- the RequireJavaVersion enforcer rule is not tripped. -->
+ <!-- paimon-eslib and its external ESLib/Lucene dependencies
require Java 11+. Keep
+ this module opt-in so the default reactor is independent of
the external ESLib
+ repository. Build it explicitly with -Ppaimon-eslib on JDK
11+. -->
<id>paimon-eslib</id>
<modules>
<module>paimon-eslib</module>
</modules>
- <activation>
- <jdk>[11,)</jdk>
- </activation>
</profile>
<profile>
