This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/paimon-full-text.git
The following commit(s) were added to refs/heads/main by this push:
new e0fef70 [core] Support Paimon full-text query DSL (#6)
e0fef70 is described below
commit e0fef700a9789f7ccc9f1f4e92e4235fa2209fd0
Author: Jingsong Lee <[email protected]>
AuthorDate: Sun Jul 5 10:48:21 2026 +0800
[core] Support Paimon full-text query DSL (#6)
---
core/src/index.rs | 58 +++++++++++++++++++++++++----
core/src/query.rs | 87 ++++++++++++++++++++++++++++++++++++++++++--
core/tests/core_roundtrip.rs | 54 +++++++++++++++++++++++++++
3 files changed, 188 insertions(+), 11 deletions(-)
diff --git a/core/src/index.rs b/core/src/index.rs
index f358f34..e32773a 100644
--- a/core/src/index.rs
+++ b/core/src/index.rs
@@ -329,8 +329,12 @@ fn build_query(
terms,
operator,
boost,
+ fuzziness,
+ max_expansions,
+ prefix_length,
} => {
validate_column(config, column)?;
+ validate_match_options(*fuzziness, *max_expansions,
*prefix_length)?;
let text_field = index
.schema()
.get_field(&config.text_field)
@@ -374,13 +378,28 @@ fn build_query(
.parse_query(&query_text)
.map_err(|e| FtIndexError::InvalidQuery(e.to_string()))
}
- FullTextQuery::Boolean { queries } => {
- if queries.is_empty() {
+ FullTextQuery::Boolean {
+ should,
+ must,
+ must_not,
+ queries,
+ } => {
+ if should.is_empty() && must.is_empty() && must_not.is_empty() &&
queries.is_empty() {
return Err(FtIndexError::InvalidQuery(
"boolean query must contain at least one
clause".to_string(),
));
}
- let mut children = Vec::with_capacity(queries.len());
+ let mut children =
+ Vec::with_capacity(should.len() + must.len() + must_not.len()
+ queries.len());
+ for child in should {
+ children.push((Occur::Should, build_query(index, config,
child)?));
+ }
+ for child in must {
+ children.push((Occur::Must, build_query(index, config,
child)?));
+ }
+ for child in must_not {
+ children.push((Occur::MustNot, build_query(index, config,
child)?));
+ }
for (occur, child) in queries {
let occur = match occur {
BooleanOccur::Should => Occur::Should,
@@ -406,6 +425,29 @@ fn build_query(
}
}
+fn validate_match_options(
+ fuzziness: Option<u8>,
+ max_expansions: usize,
+ prefix_length: u32,
+) -> Result<()> {
+ if fuzziness.unwrap_or(0) != 0 {
+ return Err(FtIndexError::InvalidQuery(
+ "match query fuzziness is not supported by paimon-full-text
yet".to_string(),
+ ));
+ }
+ if max_expansions != 50 {
+ return Err(FtIndexError::InvalidQuery(
+ "match query max_expansions is not supported by paimon-full-text
yet".to_string(),
+ ));
+ }
+ if prefix_length != 0 {
+ return Err(FtIndexError::InvalidQuery(
+ "match query prefix_length is not supported by paimon-full-text
yet".to_string(),
+ ));
+ }
+ Ok(())
+}
+
fn validate_negative_boost(negative_boost: f32) -> Result<()> {
if !negative_boost.is_finite() || !(0.0..=1.0).contains(&negative_boost) {
return Err(FtIndexError::InvalidQuery(format!(
@@ -560,12 +602,12 @@ fn matches_doc(docset: &mut dyn DocSet, doc: DocId) ->
bool {
}
}
-fn validate_column(config: &FullTextIndexConfig, column: &str) -> Result<()> {
- if column == config.text_field {
+fn validate_column(_config: &FullTextIndexConfig, column: &str) -> Result<()> {
+ if !column.trim().is_empty() {
Ok(())
} else {
- Err(FtIndexError::InvalidQuery(format!(
- "unknown full-text column '{column}'"
- )))
+ Err(FtIndexError::InvalidQuery(
+ "full-text query column must not be empty".to_string(),
+ ))
}
}
diff --git a/core/src/query.rs b/core/src/query.rs
index 1b1d0a6..93ce35e 100644
--- a/core/src/query.rs
+++ b/core/src/query.rs
@@ -1,38 +1,91 @@
use crate::error::{FtIndexError, Result};
-use serde::{Deserialize, Serialize};
+use serde::de::Error as DeError;
+use serde::{Deserialize, Deserializer, Serialize};
-#[derive(Clone, Copy, Debug, Default, Serialize, Deserialize, PartialEq, Eq)]
+#[derive(Clone, Copy, Debug, Default, Serialize, PartialEq, Eq)]
pub enum MatchOperator {
#[default]
Or,
And,
}
-#[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq, Eq)]
+impl<'de> Deserialize<'de> for MatchOperator {
+ fn deserialize<D>(deserializer: D) -> std::result::Result<Self, D::Error>
+ where
+ D: Deserializer<'de>,
+ {
+ let value = String::deserialize(deserializer)?;
+ match value.trim() {
+ "Or" | "or" | "OR" => Ok(Self::Or),
+ "And" | "and" | "AND" => Ok(Self::And),
+ _ => Err(DeError::custom(format!(
+ "invalid full-text query operator: {value}"
+ ))),
+ }
+ }
+}
+
+#[derive(Clone, Copy, Debug, Serialize, PartialEq, Eq)]
pub enum BooleanOccur {
Should,
Must,
MustNot,
}
+impl<'de> Deserialize<'de> for BooleanOccur {
+ fn deserialize<D>(deserializer: D) -> std::result::Result<Self, D::Error>
+ where
+ D: Deserializer<'de>,
+ {
+ let value = String::deserialize(deserializer)?;
+ match value.trim() {
+ "Should" | "should" | "SHOULD" => Ok(Self::Should),
+ "Must" | "must" | "MUST" => Ok(Self::Must),
+ "MustNot" | "must_not" | "MUST_NOT" | "mustnot" | "MUSTNOT" =>
Ok(Self::MustNot),
+ _ => Err(DeError::custom(format!(
+ "invalid boolean query occur: {value}"
+ ))),
+ }
+ }
+}
+
#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
#[serde(rename_all = "snake_case")]
pub enum FullTextQuery {
Match {
column: String,
+ #[serde(alias = "query")]
terms: String,
#[serde(default)]
operator: MatchOperator,
#[serde(default = "default_boost")]
boost: f32,
+ #[serde(
+ default = "default_fuzziness",
+ deserialize_with = "deserialize_fuzziness"
+ )]
+ fuzziness: Option<u8>,
+ #[serde(default = "default_max_expansions", alias = "maxExpansions")]
+ max_expansions: usize,
+ #[serde(default, alias = "prefixLength")]
+ prefix_length: u32,
},
+ #[serde(alias = "phrase")]
MatchPhrase {
column: String,
+ #[serde(alias = "query")]
terms: String,
#[serde(default)]
slop: u32,
},
Boolean {
+ #[serde(default)]
+ should: Vec<FullTextQuery>,
+ #[serde(default)]
+ must: Vec<FullTextQuery>,
+ #[serde(default)]
+ must_not: Vec<FullTextQuery>,
+ #[serde(default)]
queries: Vec<(BooleanOccur, FullTextQuery)>,
},
Boost {
@@ -51,6 +104,31 @@ fn default_negative_boost() -> f32 {
0.5
}
+fn default_fuzziness() -> Option<u8> {
+ Some(0)
+}
+
+fn default_max_expansions() -> usize {
+ 50
+}
+
+fn deserialize_fuzziness<'de, D>(deserializer: D) ->
std::result::Result<Option<u8>, D::Error>
+where
+ D: Deserializer<'de>,
+{
+ let value = Option::<serde_json::Value>::deserialize(deserializer)?;
+ match value {
+ None | Some(serde_json::Value::Null) => Ok(None),
+ Some(serde_json::Value::String(value)) if
value.eq_ignore_ascii_case("auto") => Ok(None),
+ Some(serde_json::Value::Number(value)) => value
+ .as_u64()
+ .and_then(|value| u8::try_from(value).ok())
+ .map(Some)
+ .ok_or_else(|| DeError::custom("fuzziness must be an unsigned
byte")),
+ Some(value) => Err(DeError::custom(format!("invalid fuzziness:
{value}"))),
+ }
+}
+
impl FullTextQuery {
pub fn match_query(terms: impl Into<String>, column: impl Into<String>) ->
Self {
Self::Match {
@@ -58,6 +136,9 @@ impl FullTextQuery {
terms: terms.into(),
operator: MatchOperator::Or,
boost: 1.0,
+ fuzziness: Some(0),
+ max_expansions: 50,
+ prefix_length: 0,
}
}
diff --git a/core/tests/core_roundtrip.rs b/core/tests/core_roundtrip.rs
index 02a6087..0ad7347 100644
--- a/core/tests/core_roundtrip.rs
+++ b/core/tests/core_roundtrip.rs
@@ -102,6 +102,9 @@ fn match_query_and_operator_filters_terms() ->
anyhow::Result<()> {
terms: "paimon indexes".to_string(),
operator: MatchOperator::And,
boost: 1.0,
+ fuzziness: Some(0),
+ max_expansions: 50,
+ prefix_length: 0,
};
let result = reader.search(query, 10)?;
@@ -316,3 +319,54 @@ fn query_json_round_trip() -> anyhow::Result<()> {
assert_eq!(parsed, query);
Ok(())
}
+
+#[test]
+fn parse_paimon_match_query_json_aliases() -> anyhow::Result<()> {
+ let query = FullTextQuery::from_json(
+
r#"{"match":{"column":"text","query":"paimon","operator":"AND","boost":2.0,"fuzziness":0,"maxExpansions":50,"prefixLength":0}}"#,
+ )?;
+
+ assert_eq!(
+ query,
+ FullTextQuery::Match {
+ column: "text".to_string(),
+ terms: "paimon".to_string(),
+ operator: MatchOperator::And,
+ boost: 2.0,
+ fuzziness: Some(0),
+ max_expansions: 50,
+ prefix_length: 0,
+ }
+ );
+ Ok(())
+}
+
+#[test]
+fn search_accepts_paimon_logical_column_name() -> anyhow::Result<()> {
+ let bytes = build_index()?;
+ let mut reader = FullTextIndexReader::open(SliceReader::new(bytes))?;
+ let query = FullTextQuery::from_json(
+ r#"{"match":{"column":"content","query":"paimon","operator":"OR"}}"#,
+ )?;
+ let result = reader.search(query, 10)?;
+
+ assert_eq!(result.row_ids.len(), 2);
+ assert!(result.row_ids.contains(&10));
+ assert!(result.row_ids.contains(&12));
+ Ok(())
+}
+
+#[test]
+fn parse_paimon_boolean_query_json_forms() -> anyhow::Result<()> {
+ let query = FullTextQuery::from_json(
+
r#"{"boolean":{"must":[{"match":{"column":"text","terms":"paimon"}}],"must_not":[{"phrase":{"column":"text","query":"legacy"}}]}}"#,
+ )?;
+ let bytes = build_index()?;
+ let mut reader = FullTextIndexReader::open(SliceReader::new(bytes))?;
+ let result = reader.search(query, 10)?;
+
+ assert_eq!(result.row_ids.len(), 2);
+ assert!(result.row_ids.contains(&10));
+ assert!(result.row_ids.contains(&12));
+ Ok(())
+}
