This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/paimon-full-text.git
The following commit(s) were added to refs/heads/main by this push:
new 85a4121 Add Jieba tokenizer support (#4)
85a4121 is described below
commit 85a41217896399c13c56048e628acc12f0ce1be1
Author: Jingsong Lee <[email protected]>
AuthorDate: Sun Jul 5 08:40:39 2026 +0800
Add Jieba tokenizer support (#4)
---
Cargo.lock | 619 +++++++++++++++-------
Cargo.toml | 5 +-
README.md | 2 +-
core/Cargo.toml | 1 +
core/src/index.rs | 16 +-
core/src/tokenizer.rs | 10 +
core/tests/core_roundtrip.rs | 64 +++
docs/designs/2026-07-04-paimon-full-text-index.md | 10 +-
docs/paimon-integration.md | 2 +
9 files changed, 521 insertions(+), 208 deletions(-)
diff --git a/Cargo.lock b/Cargo.lock
index ccb6369..b5b06ce 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1,6 +1,12 @@
# This file is automatically @generated by Cargo.
# It is not intended for manual editing.
-version = 3
+version = 4
+
+[[package]]
+name = "adler32"
+version = "1.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "aae1277d39aeec15cb388266ecc24b11c80469deae6067e17a1a7aa9e5c1f234"
[[package]]
name = "aho-corasick"
@@ -70,12 +76,43 @@ dependencies = [
"crunchy",
]
+[[package]]
+name = "bon"
+version = "3.9.3"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "a602c73c7b0148ec6d12af6fd5cc7a46e2eacc8878271a999abac56eed12f561"
+dependencies = [
+ "bon-macros",
+ "rustversion",
+]
+
+[[package]]
+name = "bon-macros"
+version = "3.9.3"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "6dee98b0db6a962de883bf5d20362dee4d7ca0d12fe39a7c6c73c844e1cd7c1f"
+dependencies = [
+ "darling",
+ "ident_case",
+ "prettyplease",
+ "proc-macro2",
+ "quote",
+ "rustversion",
+ "syn",
+]
+
[[package]]
name = "bumpalo"
version = "3.20.3"
source = "registry+https://github.com/rust-lang/crates.io-index";
checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649"
+[[package]]
+name = "bytecount"
+version = "0.6.9"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "175812e0be2bccb6abe50bb8d566126198344f707e304f45c648fd8f2cc0365e"
+
[[package]]
name = "bytemuck"
version = "1.25.0"
@@ -106,6 +143,15 @@ dependencies = [
"shlex",
]
+[[package]]
+name = "cedarwood"
+version = "0.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "c0524a528a6a0288df1863c3c20fe92c301875b4941e7b6c4b394ab08c5a4c55"
+dependencies = [
+ "smallvec",
+]
+
[[package]]
name = "census"
version = "0.4.2"
@@ -183,6 +229,52 @@ version = "0.2.4"
source = "registry+https://github.com/rust-lang/crates.io-index";
checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5"
+[[package]]
+name = "darling"
+version = "0.23.0"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "25ae13da2f202d56bd7f91c25fba009e7717a1e4a1cc98a76d844b65ae912e9d"
+dependencies = [
+ "darling_core",
+ "darling_macro",
+]
+
+[[package]]
+name = "darling_core"
+version = "0.23.0"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "9865a50f7c335f53564bb694ef660825eb8610e0a53d3e11bf1b0d3df31e03b0"
+dependencies = [
+ "ident_case",
+ "proc-macro2",
+ "quote",
+ "strsim",
+ "syn",
+]
+
+[[package]]
+name = "darling_macro"
+version = "0.23.0"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d"
+dependencies = [
+ "darling_core",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "dary_heap"
+version = "0.3.9"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "8b1e3a325bc115f096c8b77bbf027a7c2592230e70be2d985be950d3d5e60ebe"
+
+[[package]]
+name = "datasketches"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "c286de4e81ea2590afc24d754e0f83810c566f50a1388fa75ebd57928c0d9745"
+
[[package]]
name = "deranged"
version = "0.5.8"
@@ -194,9 +286,9 @@ dependencies = [
[[package]]
name = "downcast-rs"
-version = "1.2.1"
+version = "2.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "75b325c5dbd37f80359721ad39aca5a29fb04c89279657cffdda8736d0c0b9d2"
+checksum = "117240f60069e65410b3ae1bb213295bd828f707b5bec6596a1afc8793ce0cbc"
[[package]]
name = "either"
@@ -210,6 +302,17 @@ version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index";
checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f"
+[[package]]
+name = "erased-serde"
+version = "0.4.10"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "d2add8a07dd6a8d93ff627029c51de145e12686fbc36ecb298ac22e74cf02dec"
+dependencies = [
+ "serde",
+ "serde_core",
+ "typeid",
+]
+
[[package]]
name = "errno"
version = "0.3.14"
@@ -246,18 +349,18 @@ checksum =
"3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"
[[package]]
name = "foldhash"
-version = "0.1.5"
+version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2"
+checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb"
[[package]]
name = "fs4"
-version = "0.8.4"
+version = "0.13.1"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "f7e180ac76c23b45e767bd7ae9579bc0bb458618c4bc71835926e098e61d15f8"
+checksum = "8640e34b88f7652208ce9e88b1a37a2ae95227d84abec377ccd3c5cfeb141ed4"
dependencies = [
- "rustix 0.38.44",
- "windows-sys 0.52.0",
+ "rustix",
+ "windows-sys 0.59.0",
]
[[package]]
@@ -266,6 +369,17 @@ version = "0.3.32"
source = "registry+https://github.com/rust-lang/crates.io-index";
checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d"
+[[package]]
+name = "futures-macro"
+version = "0.3.32"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
[[package]]
name = "futures-task"
version = "0.3.32"
@@ -279,22 +393,12 @@ source =
"registry+https://github.com/rust-lang/crates.io-index";
checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6"
dependencies = [
"futures-core",
+ "futures-macro",
"futures-task",
"pin-project-lite",
"slab",
]
-[[package]]
-name = "getrandom"
-version = "0.2.17"
-source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0"
-dependencies = [
- "cfg-if",
- "libc",
- "wasi",
-]
-
[[package]]
name = "getrandom"
version = "0.3.4"
@@ -320,21 +424,15 @@ dependencies = [
[[package]]
name = "hashbrown"
-version = "0.15.5"
+version = "0.16.1"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1"
+checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100"
dependencies = [
"allocator-api2",
"equivalent",
"foldhash",
]
-[[package]]
-name = "hermit-abi"
-version = "0.5.2"
-source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c"
-
[[package]]
name = "htmlescape"
version = "0.3.1"
@@ -342,22 +440,58 @@ source =
"registry+https://github.com/rust-lang/crates.io-index";
checksum = "e9025058dae765dee5070ec375f591e2ba14638c63feff74f13805a72e523163"
[[package]]
-name = "instant"
-version = "0.1.13"
+name = "ident_case"
+version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "e0242819d153cba4b4b05a5a8f2a7e9bbf97b6055b2a002b395c96b5ff3c0222"
+checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39"
+
+[[package]]
+name = "include-flate"
+version = "0.3.3"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "23e233413926ef735f7d87024466cfda5a4b87467730846bd82ea7d504121347"
dependencies = [
- "cfg-if",
- "js-sys",
- "wasm-bindgen",
- "web-sys",
+ "include-flate-codegen",
+ "include-flate-compress",
+]
+
+[[package]]
+name = "include-flate-codegen"
+version = "0.3.3"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "5e7148f24ef8922cc0e5574ebb908729ccdd3a110c440a45165733fedadd9969"
+dependencies = [
+ "include-flate-compress",
+ "proc-macro-error2",
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "include-flate-compress"
+version = "0.3.3"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "74783a9ed407e844e99d5e7a57bd650acbfa124cf6e97ffd790ba59d8ab8e7ff"
+dependencies = [
+ "libflate",
+ "zstd",
+]
+
+[[package]]
+name = "inventory"
+version = "0.3.24"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "a4f0c30c76f2f4ccee3fe55a2435f691ca00c0e4bd87abe4f4a851b1d4dac39b"
+dependencies = [
+ "rustversion",
]
[[package]]
name = "itertools"
-version = "0.12.1"
+version = "0.14.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569"
+checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285"
dependencies = [
"either",
]
@@ -368,6 +502,30 @@ version = "1.0.18"
source = "registry+https://github.com/rust-lang/crates.io-index";
checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682"
+[[package]]
+name = "jieba-macros"
+version = "0.10.1"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "46adade69b634535a8f495cf87710ed893cff53e1dbc9dd750c2ab81c5defb82"
+dependencies = [
+ "phf_codegen",
+]
+
+[[package]]
+name = "jieba-rs"
+version = "0.10.1"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "11b53580aaa8ec8b713da271da434f8947409242c537a9ab3f7b76bdbb19e8a9"
+dependencies = [
+ "bytecount",
+ "cedarwood",
+ "include-flate",
+ "jieba-macros",
+ "phf",
+ "regex",
+ "rustc-hash",
+]
+
[[package]]
name = "jni"
version = "0.21.1"
@@ -379,7 +537,7 @@ dependencies = [
"combine",
"jni-sys 0.3.1",
"log",
- "thiserror",
+ "thiserror 1.0.69",
"walkdir",
"windows-sys 0.45.0",
]
@@ -433,6 +591,12 @@ dependencies = [
"wasm-bindgen",
]
+[[package]]
+name = "lazy_static"
+version = "1.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe"
+
[[package]]
name = "levenshtein_automata"
version = "0.2.1"
@@ -446,16 +610,28 @@ source =
"registry+https://github.com/rust-lang/crates.io-index";
checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66"
[[package]]
-name = "libm"
-version = "0.2.16"
+name = "libflate"
+version = "2.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981"
+checksum = "cd96e993e5f3368b0cb8497dae6c860c22af8ff18388c61c6c0b86c58d86b5df"
+dependencies = [
+ "adler32",
+ "crc32fast",
+ "dary_heap",
+ "libflate_lz77",
+ "no_std_io2",
+]
[[package]]
-name = "linux-raw-sys"
-version = "0.4.15"
+name = "libflate_lz77"
+version = "2.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab"
+checksum = "ff7a10e427698aef6eef269482776debfef63384d30f13aad39a1a95e0e098fd"
+dependencies = [
+ "hashbrown",
+ "no_std_io2",
+ "rle-decode-fast",
+]
[[package]]
name = "linux-raw-sys"
@@ -471,26 +647,25 @@ checksum =
"0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad"
[[package]]
name = "lru"
-version = "0.12.5"
+version = "0.16.4"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "234cf4f4a04dc1f57e24b96cc0cd600cf2af460d4161ac5ecdd0af8e1f3b2a38"
+checksum = "7f66e8d5d03f609abc3a39e6f08e4164ebf1447a732906d39eb9b99b7919ef39"
dependencies = [
"hashbrown",
]
[[package]]
name = "lz4_flex"
-version = "0.11.6"
+version = "0.13.1"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "373f5eceeeab7925e0c1098212f2fbc4d416adec9d35051a6ab251e824c1854a"
+checksum = "7ef0d4ed8669f8f8826eb00dc878084aa8f253506c4fd5e8f58f5bce72ddb97e"
[[package]]
name = "measure_time"
-version = "0.8.3"
+version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "dbefd235b0aadd181626f281e1d684e116972988c14c264e42069d5e8a5775cc"
+checksum = "51c55d61e72fc3ab704396c5fa16f4c184db37978ae4e94ca8959693a235fc0e"
dependencies = [
- "instant",
"log",
]
@@ -521,6 +696,15 @@ version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index";
checksum = "2195bf6aa996a481483b29d62a7663eed3fe39600c460e323f8ff41e90bdd89b"
+[[package]]
+name = "no_std_io2"
+version = "0.9.4"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "418abd1b6d34fbf6cae440dc874771b0525a604428704c76e48b29a5e67b8003"
+dependencies = [
+ "memchr",
+]
+
[[package]]
name = "nom"
version = "7.1.3"
@@ -544,17 +728,6 @@ source =
"registry+https://github.com/rust-lang/crates.io-index";
checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841"
dependencies = [
"autocfg",
- "libm",
-]
-
-[[package]]
-name = "num_cpus"
-version = "1.17.0"
-source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "91df4bbde75afed763b708b7eee1e8e7651e02d97f6d5dd763e89367e957b23b"
-dependencies = [
- "hermit-abi",
- "libc",
]
[[package]]
@@ -569,11 +742,20 @@ version = "0.1.13"
source = "registry+https://github.com/rust-lang/crates.io-index";
checksum = "269bca4c2591a28585d6bf10d9ed0332b7d76900a1b02bec41bdc3a2cdcda107"
+[[package]]
+name = "ordered-float"
+version = "5.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "b7d950ca161dc355eaf28f82b11345ed76c6e1f6eb1f4f4479e0323b9e2fbd0e"
+dependencies = [
+ "num-traits",
+]
+
[[package]]
name = "ownedbytes"
-version = "0.7.0"
+version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "c3a059efb063b8f425b948e042e6b9bd85edfe60e913630ed727b23e2dfcc558"
+checksum = "2fbd56f7631767e61784dc43f8580f403f4475bd4aaa4da003e6295e1bab4a7e"
dependencies = [
"stable_deref_trait",
]
@@ -587,8 +769,9 @@ dependencies = [
"serde",
"serde_json",
"tantivy",
+ "tantivy-jieba",
"tempfile",
- "thiserror",
+ "thiserror 1.0.69",
]
[[package]]
@@ -611,6 +794,45 @@ dependencies = [
"serde_json",
]
+[[package]]
+name = "phf"
+version = "0.13.1"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "c1562dc717473dbaa4c1f85a36410e03c047b2e7df7f45ee938fbef64ae7fadf"
+dependencies = [
+ "phf_shared",
+ "serde",
+]
+
+[[package]]
+name = "phf_codegen"
+version = "0.13.1"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "49aa7f9d80421bca176ca8dbfebe668cc7a2684708594ec9f3c0db0805d5d6e1"
+dependencies = [
+ "phf_generator",
+ "phf_shared",
+]
+
+[[package]]
+name = "phf_generator"
+version = "0.13.1"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "135ace3a761e564ec88c03a77317a7c6b80bb7f7135ef2544dbe054243b89737"
+dependencies = [
+ "fastrand",
+ "phf_shared",
+]
+
+[[package]]
+name = "phf_shared"
+version = "0.13.1"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "e57fef6bc5981e38c2ce2d63bfa546861309f875b8a75f092d1d54ae2d64f266"
+dependencies = [
+ "siphasher",
+]
+
[[package]]
name = "pin-project-lite"
version = "0.2.17"
@@ -630,12 +852,35 @@ source =
"registry+https://github.com/rust-lang/crates.io-index";
checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391"
[[package]]
-name = "ppv-lite86"
-version = "0.2.21"
+name = "prettyplease"
+version = "0.2.37"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9"
+checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b"
dependencies = [
- "zerocopy",
+ "proc-macro2",
+ "syn",
+]
+
+[[package]]
+name = "proc-macro-error-attr2"
+version = "2.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "96de42df36bb9bba5542fe9f1a054b8cc87e172759a1868aa05c1f3acc89dfc5"
+dependencies = [
+ "proc-macro2",
+ "quote",
+]
+
+[[package]]
+name = "proc-macro-error2"
+version = "2.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "11ec05c52be0a07b08061f7dd003e7d7092e0472bc731b4af7bb1ef876109802"
+dependencies = [
+ "proc-macro-error-attr2",
+ "proc-macro2",
+ "quote",
+ "syn",
]
[[package]]
@@ -668,46 +913,6 @@ version = "6.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf"
-[[package]]
-name = "rand"
-version = "0.8.6"
-source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "5ca0ecfa931c29007047d1bc58e623ab12e5590e8c7cc53200d5202b69266d8a"
-dependencies = [
- "libc",
- "rand_chacha",
- "rand_core",
-]
-
-[[package]]
-name = "rand_chacha"
-version = "0.3.1"
-source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88"
-dependencies = [
- "ppv-lite86",
- "rand_core",
-]
-
-[[package]]
-name = "rand_core"
-version = "0.6.4"
-source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c"
-dependencies = [
- "getrandom 0.2.17",
-]
-
-[[package]]
-name = "rand_distr"
-version = "0.4.3"
-source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "32cb0b9bc82b0a0876c2dd994a7e7a2683d3e7390ca40e6886785ef0c7e3ee31"
-dependencies = [
- "num-traits",
- "rand",
-]
-
[[package]]
name = "rayon"
version = "1.12.0"
@@ -757,6 +962,12 @@ version = "0.8.11"
source = "registry+https://github.com/rust-lang/crates.io-index";
checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4"
+[[package]]
+name = "rle-decode-fast"
+version = "1.0.3"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "3582f63211428f83597b51b2ddb88e2a91a9d52d12831f9d08f5e624e8977422"
+
[[package]]
name = "roaring"
version = "0.11.4"
@@ -779,22 +990,9 @@ dependencies = [
[[package]]
name = "rustc-hash"
-version = "1.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2"
-
-[[package]]
-name = "rustix"
-version = "0.38.44"
+version = "2.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154"
-dependencies = [
- "bitflags",
- "errno",
- "libc",
- "linux-raw-sys 0.4.15",
- "windows-sys 0.59.0",
-]
+checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d"
[[package]]
name = "rustix"
@@ -805,7 +1003,7 @@ dependencies = [
"bitflags",
"errno",
"libc",
- "linux-raw-sys 0.12.1",
+ "linux-raw-sys",
"windows-sys 0.61.2",
]
@@ -873,11 +1071,17 @@ version = "2.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index";
checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba"
+[[package]]
+name = "siphasher"
+version = "1.0.3"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "8ee5873ec9cce0195efcb7a4e9507a04cd49aec9c83d0389df45b1ef7ba2e649"
+
[[package]]
name = "sketches-ddsketch"
-version = "0.2.2"
+version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "85636c14b73d81f541e525f585c0a2109e6744e1565b5c1668e31c70c10ed65c"
+checksum = "05e40b6cf54d988dc1a2223531b969c9a9e30906ad90ef64890c27b4bfbb46ea"
dependencies = [
"serde",
]
@@ -900,6 +1104,12 @@ version = "1.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index";
checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596"
+[[package]]
+name = "strsim"
+version = "0.11.1"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f"
+
[[package]]
name = "syn"
version = "2.0.118"
@@ -913,18 +1123,20 @@ dependencies = [
[[package]]
name = "tantivy"
-version = "0.22.1"
+version = "0.26.1"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "96599ea6fccd844fc833fed21d2eecac2e6a7c1afd9e044057391d78b1feb141"
+checksum = "edde6a10743fff00a4e1a8c9ef020bf5f3cbad301b7d2d39f2b07f123c4eac07"
dependencies = [
"aho-corasick",
"arc-swap",
"base64",
"bitpacking",
+ "bon",
"byteorder",
"census",
"crc32fast",
"crossbeam-channel",
+ "datasketches",
"downcast-rs",
"fastdivide",
"fnv",
@@ -937,7 +1149,6 @@ dependencies = [
"lz4_flex",
"measure_time",
"memmap2",
- "num_cpus",
"once_cell",
"oneshot",
"rayon",
@@ -956,26 +1167,27 @@ dependencies = [
"tantivy-stacker",
"tantivy-tokenizer-api",
"tempfile",
- "thiserror",
+ "thiserror 2.0.18",
"time",
+ "typetag",
"uuid",
"winapi",
]
[[package]]
name = "tantivy-bitpacker"
-version = "0.6.0"
+version = "0.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "284899c2325d6832203ac6ff5891b297fc5239c3dc754c5bc1977855b23c10df"
+checksum = "4fed3d674429bcd2de5d0a6d1aa5495fed8afd9c5ecce993019caf7615f53fa4"
dependencies = [
"bitpacking",
]
[[package]]
name = "tantivy-columnar"
-version = "0.3.0"
+version = "0.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "12722224ffbe346c7fec3275c699e508fd0d4710e629e933d5736ec524a1f44e"
+checksum = "c57166f5bcfd478f370ab8445afb4678dce44801fa5ce5c451aaf8595583c5dc"
dependencies = [
"downcast-rs",
"fastdivide",
@@ -989,9 +1201,9 @@ dependencies = [
[[package]]
name = "tantivy-common"
-version = "0.7.0"
+version = "0.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "8019e3cabcfd20a1380b491e13ff42f57bb38bf97c3d5fa5c07e50816e0621f4"
+checksum = "bbf10915aa75da3c3b0d58b58853d2e889efbaf32d4982a4c3715dde6bba23e5"
dependencies = [
"async-trait",
"byteorder",
@@ -1011,21 +1223,38 @@ dependencies = [
"utf8-ranges",
]
+[[package]]
+name = "tantivy-jieba"
+version = "0.20.0"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "3392170e86f1c387170aba7d171a466ffdc98a8b55b006e19ac64b123a7b690a"
+dependencies = [
+ "jieba-rs",
+ "lazy_static",
+ "tantivy-tokenizer-api",
+]
+
[[package]]
name = "tantivy-query-grammar"
-version = "0.22.0"
+version = "0.26.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "847434d4af57b32e309f4ab1b4f1707a6c566656264caa427ff4285c4d9d0b82"
+checksum = "dfadb8526b6da90704feb293b0701a6aae62ea14983143344be2dc5ce30f1d82"
dependencies = [
+ "fnv",
"nom",
+ "ordered-float",
+ "serde",
+ "serde_json",
]
[[package]]
name = "tantivy-sstable"
-version = "0.3.0"
+version = "0.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "c69578242e8e9fc989119f522ba5b49a38ac20f576fc778035b96cc94f41f98e"
+checksum = "8a2cfc3ac5164cbadc28965ffb145a8f47582a60ae5897859ad8d4316596c606"
dependencies = [
+ "futures-util",
+ "itertools",
"tantivy-bitpacker",
"tantivy-common",
"tantivy-fst",
@@ -1034,20 +1263,19 @@ dependencies = [
[[package]]
name = "tantivy-stacker"
-version = "0.3.0"
+version = "0.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "c56d6ff5591fc332739b3ce7035b57995a3ce29a93ffd6012660e0949c956ea8"
+checksum = "6cbb051742da9d53ca9e8fff43a9b10e319338b24e2c0e15d0372df19ffeb951"
dependencies = [
"murmurhash32",
- "rand_distr",
"tantivy-common",
]
[[package]]
name = "tantivy-tokenizer-api"
-version = "0.3.0"
+version = "0.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "2a0dcade25819a89cfe6f17d932c9cedff11989936bf6dd4f336d50392053b04"
+checksum = "eac258c2c6390673f2685813afeeafcb8c4e0ee7de8dd3fc46838dcc37263f98"
dependencies = [
"serde",
]
@@ -1061,7 +1289,7 @@ dependencies = [
"fastrand",
"getrandom 0.4.3",
"once_cell",
- "rustix 1.1.4",
+ "rustix",
"windows-sys 0.61.2",
]
@@ -1071,7 +1299,16 @@ version = "1.0.69"
source = "registry+https://github.com/rust-lang/crates.io-index";
checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52"
dependencies = [
- "thiserror-impl",
+ "thiserror-impl 1.0.69",
+]
+
+[[package]]
+name = "thiserror"
+version = "2.0.18"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4"
+dependencies = [
+ "thiserror-impl 2.0.18",
]
[[package]]
@@ -1085,6 +1322,17 @@ dependencies = [
"syn",
]
+[[package]]
+name = "thiserror-impl"
+version = "2.0.18"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
[[package]]
name = "time"
version = "0.3.53"
@@ -1115,6 +1363,36 @@ dependencies = [
"time-core",
]
+[[package]]
+name = "typeid"
+version = "1.0.3"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "bc7d623258602320d5c55d1bc22793b57daff0ec7efc270ea7d55ce1d5f5471c"
+
+[[package]]
+name = "typetag"
+version = "0.2.22"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "c5a897b12c6c1151ad0b138b8db50252dc301f93bc3b027db05eec82aeed298c"
+dependencies = [
+ "erased-serde",
+ "inventory",
+ "once_cell",
+ "serde",
+ "typetag-impl",
+]
+
+[[package]]
+name = "typetag-impl"
+version = "0.2.22"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "cf808357c6ed7e13ba0f3277ec8d8f21b2d501274895104263985330c726c1c5"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
[[package]]
name = "unicode-ident"
version = "1.0.24"
@@ -1149,12 +1427,6 @@ dependencies = [
"winapi-util",
]
-[[package]]
-name = "wasi"
-version = "0.11.1+wasi-snapshot-preview1"
-source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b"
-
[[package]]
name = "wasip2"
version = "1.0.4+wasi-0.2.12"
@@ -1209,16 +1481,6 @@ dependencies = [
"unicode-ident",
]
-[[package]]
-name = "web-sys"
-version = "0.3.103"
-source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "8622dcb61c0bcc9fffa6938bed81210af2da9a7e4a1a834b2e37a59b6dfb6141"
-dependencies = [
- "js-sys",
- "wasm-bindgen",
-]
-
[[package]]
name = "winapi"
version = "0.3.9"
@@ -1265,15 +1527,6 @@ dependencies = [
"windows-targets 0.42.2",
]
-[[package]]
-name = "windows-sys"
-version = "0.52.0"
-source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d"
-dependencies = [
- "windows-targets 0.52.6",
-]
-
[[package]]
name = "windows-sys"
version = "0.59.0"
@@ -1419,26 +1672,6 @@ version = "0.57.1"
source = "registry+https://github.com/rust-lang/crates.io-index";
checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e"
-[[package]]
-name = "zerocopy"
-version = "0.8.52"
-source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "ce1022995ff5ff5d841ad7d994facc23098cd40152f2c1d11cd607c6f530653f"
-dependencies = [
- "zerocopy-derive",
-]
-
-[[package]]
-name = "zerocopy-derive"
-version = "0.8.52"
-source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "1ae7f38b72ec2a254e2b87ef277cf2cd4fb97cbebf944faa6f33354da0867930"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn",
-]
-
[[package]]
name = "zmij"
version = "1.0.21"
diff --git a/Cargo.toml b/Cargo.toml
index 8a2e210..037ba19 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -6,7 +6,7 @@ resolver = "2"
edition = "2021"
license = "Apache-2.0"
repository = "https://github.com/apache/paimon-full-text";
-rust-version = "1.75"
+rust-version = "1.86"
[workspace.dependencies]
anyhow = "1"
@@ -14,6 +14,7 @@ jni = "0.21"
serde = { version = "1", features = ["derive"] }
serde_json = "1"
roaring = "0.11"
-tantivy = "0.22"
+tantivy = "0.26"
+tantivy-jieba = "0.20"
tempfile = "3"
thiserror = "1"
diff --git a/README.md b/README.md
index 733c7b7..1b08b12 100644
--- a/README.md
+++ b/README.md
@@ -30,10 +30,10 @@ Supported tokenizers in this first implementation:
- `whitespace`
- `raw`
- `ngram`
+- `jieba`
Reserved for follow-up:
-- `jieba`
- stemming
- built-in and custom stop-word filters
- true seek-on-demand Tantivy directory instead of loading segment files into
diff --git a/core/Cargo.toml b/core/Cargo.toml
index 1f07c33..3bf1235 100644
--- a/core/Cargo.toml
+++ b/core/Cargo.toml
@@ -11,6 +11,7 @@ roaring.workspace = true
serde.workspace = true
serde_json.workspace = true
tantivy.workspace = true
+tantivy-jieba.workspace = true
tempfile.workspace = true
thiserror.workspace = true
diff --git a/core/src/index.rs b/core/src/index.rs
index 30e7f1f..482a93f 100644
--- a/core/src/index.rs
+++ b/core/src/index.rs
@@ -19,6 +19,7 @@ use tantivy::tokenizer::{
SimpleTokenizer, TextAnalyzer, WhitespaceTokenizer,
};
use tantivy::{DocId, DocSet, Index, Score, SegmentReader, TantivyDocument,
TERMINATED};
+use tantivy_jieba::JiebaTokenizer;
use tempfile::TempDir;
#[derive(Clone, Debug, PartialEq)]
@@ -164,11 +165,11 @@ impl<R: SeekRead> FullTextIndexReader<R> {
let collector = FilterCollector::new(
row_id_field,
move |row_id: u64| filter.contains(row_id),
- TopDocs::with_limit(limit),
+ TopDocs::with_limit(limit).order_by_score(),
);
searcher.search(&tantivy_query, &collector)?
} else {
- searcher.search(&tantivy_query, &TopDocs::with_limit(limit))?
+ searcher.search(&tantivy_query,
&TopDocs::with_limit(limit).order_by_score())?
};
let mut row_ids = Vec::with_capacity(top_docs.len());
let mut scores = Vec::with_capacity(top_docs.len());
@@ -238,10 +239,6 @@ fn needs_custom_default(config: &TokenizerConfig) -> bool {
fn register_tokenizer(index: &mut Index, config: &TokenizerConfig) ->
Result<()> {
match config.tokenizer {
TokenizerKind::Default if !needs_custom_default(config) => Ok(()),
- TokenizerKind::Jieba => Err(FtIndexError::InvalidOption {
- key: "tokenizer".to_string(),
- message: "jieba tokenizer is not enabled in this first
implementation".to_string(),
- }),
_ => {
let analyzer = build_text_analyzer(config)?;
index
@@ -281,10 +278,9 @@ fn build_text_analyzer(config: &TokenizerConfig) ->
Result<TextAnalyzer> {
TextAnalyzer::builder(tokenizer).dynamic()
}
TokenizerKind::Jieba => {
- return Err(FtIndexError::InvalidOption {
- key: "tokenizer".to_string(),
- message: "jieba tokenizer is not enabled in this first
implementation".to_string(),
- })
+ let mut tokenizer =
JiebaTokenizer::with_search_mode(config.jieba_search_mode);
+ tokenizer.set_ordinal_position_mode(config.jieba_ordinal_position);
+ TextAnalyzer::builder(tokenizer).dynamic()
}
};
builder =
builder.filter_dynamic(RemoveLongFilter::limit(config.max_token_length));
diff --git a/core/src/tokenizer.rs b/core/src/tokenizer.rs
index 65635b2..783877d 100644
--- a/core/src/tokenizer.rs
+++ b/core/src/tokenizer.rs
@@ -20,6 +20,8 @@ pub struct TokenizerConfig {
pub ngram_min_gram: usize,
pub ngram_max_gram: usize,
pub ngram_prefix_only: bool,
+ pub jieba_search_mode: bool,
+ pub jieba_ordinal_position: bool,
pub lower_case: bool,
pub max_token_length: usize,
pub ascii_folding: bool,
@@ -37,6 +39,8 @@ impl Default for TokenizerConfig {
ngram_min_gram: 2,
ngram_max_gram: 2,
ngram_prefix_only: false,
+ jieba_search_mode: true,
+ jieba_ordinal_position: true,
lower_case: true,
max_token_length: 40,
ascii_folding: false,
@@ -70,6 +74,12 @@ impl TokenizerConfig {
"ngram.prefix-only" => {
config.ngram_prefix_only = parse_bool(key, value)?;
}
+ "jieba.search-mode" => {
+ config.jieba_search_mode = parse_bool(key, value)?;
+ }
+ "jieba.ordinal-position" | "jieba.ordinal-position-mode" => {
+ config.jieba_ordinal_position = parse_bool(key, value)?;
+ }
"lower-case" => {
config.lower_case = parse_bool(key, value)?;
}
diff --git a/core/tests/core_roundtrip.rs b/core/tests/core_roundtrip.rs
index 00d66ec..aa38ac3 100644
--- a/core/tests/core_roundtrip.rs
+++ b/core/tests/core_roundtrip.rs
@@ -1,8 +1,10 @@
use paimon_ftindex_core::io::{PosWriter, SliceReader};
use paimon_ftindex_core::{
FullTextIndexConfig, FullTextIndexReader, FullTextIndexWriter,
FullTextQuery, MatchOperator,
+ TokenizerConfig, TokenizerKind,
};
use roaring::RoaringTreemap;
+use std::collections::HashMap;
fn build_index() -> anyhow::Result<Vec<u8>> {
let mut writer = FullTextIndexWriter::new(FullTextIndexConfig::new())?;
@@ -141,6 +143,68 @@ fn phrase_query_uses_positions() -> anyhow::Result<()> {
Ok(())
}
+#[test]
+fn jieba_tokenizer_searches_chinese_terms() -> anyhow::Result<()> {
+ let config = FullTextIndexConfig::new().tokenizer(TokenizerConfig {
+ tokenizer: TokenizerKind::Jieba,
+ ..TokenizerConfig::default()
+ });
+ let mut writer = FullTextIndexWriter::new(config)?;
+ writer.add_document(20, "中华人民共和国人民大会堂")?;
+ writer.add_document(21, "北京大学支持全文检索")?;
+
+ let mut bytes = Vec::new();
+ writer.write(&mut PosWriter::new(&mut bytes))?;
+
+ let mut reader = FullTextIndexReader::open(SliceReader::new(bytes))?;
+ let result = reader.search(FullTextQuery::match_query("中华", "text"), 10)?;
+
+ assert_eq!(result.row_ids, vec![20]);
+ Ok(())
+}
+
+#[test]
+fn jieba_tokenizer_supports_chinese_phrase_queries() -> anyhow::Result<()> {
+ let config = FullTextIndexConfig::new().tokenizer(TokenizerConfig {
+ tokenizer: TokenizerKind::Jieba,
+ jieba_ordinal_position: true,
+ ..TokenizerConfig::default()
+ });
+ let mut writer = FullTextIndexWriter::new(config)?;
+ writer.add_document(30, "北京大学支持全文检索")?;
+ writer.add_document(31, "北京的大学很多")?;
+
+ let mut bytes = Vec::new();
+ writer.write(&mut PosWriter::new(&mut bytes))?;
+
+ let mut reader = FullTextIndexReader::open(SliceReader::new(bytes))?;
+ let result = reader.search(FullTextQuery::phrase("北京大学", "text"), 10)?;
+
+ assert_eq!(result.row_ids, vec![30]);
+ Ok(())
+}
+
+#[test]
+fn tokenizer_options_parse_jieba_settings() -> anyhow::Result<()> {
+ let mut options = HashMap::new();
+ options.insert("fulltext.tokenizer".to_string(), "jieba".to_string());
+ options.insert(
+ "fulltext.jieba.search-mode".to_string(),
+ "false".to_string(),
+ );
+ options.insert(
+ "fulltext.jieba.ordinal-position".to_string(),
+ "false".to_string(),
+ );
+
+ let config = TokenizerConfig::from_options(&options)?;
+
+ assert_eq!(config.tokenizer, TokenizerKind::Jieba);
+ assert!(!config.jieba_search_mode);
+ assert!(!config.jieba_ordinal_position);
+ Ok(())
+}
+
#[test]
fn boost_query_requires_positive_match() -> anyhow::Result<()> {
let mut writer = FullTextIndexWriter::new(FullTextIndexConfig::new())?;
diff --git a/docs/designs/2026-07-04-paimon-full-text-index.md
b/docs/designs/2026-07-04-paimon-full-text-index.md
index 02e174f..ee22556 100644
--- a/docs/designs/2026-07-04-paimon-full-text-index.md
+++ b/docs/designs/2026-07-04-paimon-full-text-index.md
@@ -106,7 +106,7 @@ Header JSON:
}
},
"document_count": 0,
- "tantivy_version": "0.22.1"
+ "tantivy_version": "0.26.1"
},
"files": [
{"name": "meta.json", "offset": 0, "length": 1234}
@@ -367,6 +367,8 @@ fulltext.tokenizer
fulltext.ngram.min-gram
fulltext.ngram.max-gram
fulltext.ngram.prefix-only
+fulltext.jieba.search-mode
+fulltext.jieba.ordinal-position
fulltext.lower-case
fulltext.max-token-length
fulltext.ascii-folding
@@ -389,10 +391,10 @@ Supported first implementation tokenizers:
- `whitespace`
- `raw`
- `ngram`
+- `jieba`
Reserved follow-up tokenizers and filters:
-- `jieba`
- stemming
- built-in stop words
- custom stop words
@@ -403,6 +405,10 @@ Supported first implementation filters:
- max token length
- ASCII folding
+The `jieba` tokenizer uses Tantivy's current external tokenizer API via
+`tantivy-jieba`. Search mode and ordinal token positions are enabled by
+default for Chinese full-text recall and phrase query correctness.
+
No dynamic Rust tokenizer plugins in v1. They complicate packaging and native
loading too much for an initial library.
diff --git a/docs/paimon-integration.md b/docs/paimon-integration.md
index e5da028..4905292 100644
--- a/docs/paimon-integration.md
+++ b/docs/paimon-integration.md
@@ -24,6 +24,8 @@ fulltext.tokenizer
fulltext.ngram.min-gram
fulltext.ngram.max-gram
fulltext.ngram.prefix-only
+fulltext.jieba.search-mode
+fulltext.jieba.ordinal-position
fulltext.lower-case
fulltext.max-token-length
fulltext.ascii-folding
