This is an automated email from the ASF dual-hosted git repository. github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/main/pr-6801-c24755c98e0d17b07712ee677ee9ddd7d33ccb68 in repository https://gitbox.apache.org/repos/asf/texera.git
commit 110a54c151dc692a0720235f3951c41e60ec363a Author: Kary Zheng <[email protected]> AuthorDate: Wed Jul 22 20:59:13 2026 -0700 fix(KeywordSearch): keep punctuation handling consistent in case-sensitive mode (#6801) ### What changes were proposed in this PR? Fixes an unintended side effect of the KeywordSearch "case sensitive" option. `KeywordSearchOpExec` selects an analyzer based on `isCaseSensitive`: - **off (default)** → `StandardAnalyzer`, whose `StandardTokenizer` splits on Unicode word boundaries and strips punctuation, so `"perfect."` tokenizes to `perfect`. - **on** → `CaseSensitiveAnalyzer`, which used a bare `WhitespaceTokenizer`. That splits **only** on whitespace, so punctuation stays glued to the token: `"...absolutely perfect."` tokenizes to `[absolutely, perfect.]`. The intent of `CaseSensitiveAnalyzer` was only to *skip lowercasing*. But swapping the tokenizer also dropped Unicode word-boundary tokenization, so enabling "case sensitive" silently changed **punctuation handling** too — a side effect the option name does not imply. A user who turns on case sensitivity to match `Trump` exactly finds that `perfect` no longer matches `perfect.`, `USA` no longer matches `USA.`, etc. The fix makes `CaseSensitiveAnalyzer` mirror `StandardAnalyzer`'s pipeline **minus** the `LowerCaseFilter`: it now uses `StandardTokenizer` (Unicode word-boundary tokenization, punctuation stripped) and omits lowercasing. Case sensitivity becomes the *only* behavioral difference from the default analyzer. ```diff -import org.apache.lucene.analysis.core.WhitespaceTokenizer +import org.apache.lucene.analysis.standard.StandardTokenizer ... - val tokenizer = new WhitespaceTokenizer() + val tokenizer = new StandardTokenizer() val stream: TokenStream = new StopFilter(tokenizer, CharArraySet.EMPTY_SET) ``` ### Any related issues, documentation, discussions? Closes #6761 Note: the standalone/Python translation of KeywordSearch (`KeywordSearchOpDesc.generateStandaloneCode`) does not implement case sensitivity at all — that is a separate gap and out of scope here. ### How was this PR tested? Added a regression test to `KeywordSearchOpExecSpec` asserting that in case-sensitive mode, keyword `perfect` matches the row `everything was absolutely perfect.` (a word adjacent to trailing punctuation). This test fails on `main` (the row is not matched) and passes with this change. Ran the full suite locally: ``` sbt "WorkflowOperator/testOnly org.apache.texera.amber.operator.keywordSearch.KeywordSearchOpExecSpec" ... Tests: succeeded 18, failed 0, canceled 0, ignored 0, pending 0 All tests passed. ``` The existing case-sensitive tests (`twitter`/`Twitter`) still pass, confirming case sensitivity itself is unchanged. ### Was this PR authored or co-authored using generative AI tooling? Generated-by: Claude Code (Claude Opus 4.8) --------- Co-authored-by: Claude Opus 4.8 (1M context) <[email protected]> --- .../keywordSearch/CaseSensitiveAnalyzer.scala | 9 +++++---- .../keywordSearch/CaseSensitiveAnalyzerSpec.scala | 22 ++++++++++++---------- .../keywordSearch/KeywordSearchOpExecSpec.scala | 18 +++++++++++++++++- 3 files changed, 34 insertions(+), 15 deletions(-) diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/keywordSearch/CaseSensitiveAnalyzer.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/keywordSearch/CaseSensitiveAnalyzer.scala index 797cf274ac..1edf087738 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/keywordSearch/CaseSensitiveAnalyzer.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/keywordSearch/CaseSensitiveAnalyzer.scala @@ -19,16 +19,17 @@ package org.apache.texera.amber.operator.keywordSearch import org.apache.lucene.analysis.{Analyzer, TokenStream} -import org.apache.lucene.analysis.core.WhitespaceTokenizer +import org.apache.lucene.analysis.standard.StandardTokenizer import org.apache.lucene.analysis.CharArraySet import org.apache.lucene.analysis.StopFilter import org.apache.lucene.analysis.Analyzer.TokenStreamComponents -// Achieves case sensitivity by skipping the lowercasing and normalization -// pipeline used in StandardAnalyzer. +// Mirrors StandardAnalyzer but omits the LowerCaseFilter, so tokens keep their +// case while still splitting on Unicode word boundaries. A bare WhitespaceTokenizer +// would instead glue punctuation to tokens (e.g. "perfect." would not match "perfect"). class CaseSensitiveAnalyzer extends Analyzer { override protected def createComponents(fieldName: String): TokenStreamComponents = { - val tokenizer = new WhitespaceTokenizer() + val tokenizer = new StandardTokenizer() val stream: TokenStream = new StopFilter(tokenizer, CharArraySet.EMPTY_SET) new TokenStreamComponents(tokenizer, stream) } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/keywordSearch/CaseSensitiveAnalyzerSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/keywordSearch/CaseSensitiveAnalyzerSpec.scala index 776db23a4c..c5a202928b 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/keywordSearch/CaseSensitiveAnalyzerSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/keywordSearch/CaseSensitiveAnalyzerSpec.scala @@ -76,11 +76,11 @@ class CaseSensitiveAnalyzerSpec extends AnyFlatSpec { } // --------------------------------------------------------------------------- - // Whitespace tokenization — the underlying tokenizer is - // WhitespaceTokenizer; pin its splitting behavior. + // Word-boundary tokenization — the underlying tokenizer is + // StandardTokenizer (Unicode UAX#29); pin its splitting behavior. // --------------------------------------------------------------------------- - "CaseSensitiveAnalyzer (whitespace tokenizer)" should + "CaseSensitiveAnalyzer (standard tokenizer)" should "split on a single space" in { assert(tokensOf("body", "a b c") == List("a", "b", "c")) } @@ -94,16 +94,18 @@ class CaseSensitiveAnalyzerSpec extends AnyFlatSpec { } // --------------------------------------------------------------------------- - // Punctuation — WhitespaceTokenizer keeps punctuation attached to - // adjacent characters (it only splits on whitespace). + // Punctuation — StandardTokenizer splits on Unicode word boundaries, so + // punctuation is a boundary and is stripped rather than kept attached. + // This is the whole point of the fix: case-sensitive matching must not + // change punctuation handling relative to the default StandardAnalyzer. // --------------------------------------------------------------------------- it should - "leave punctuation attached to tokens (WhitespaceTokenizer only splits on whitespace)" in { - // `"abc,def"` has no whitespace inside, so it stays one token. - assert(tokensOf("body", "abc,def") == List("abc,def")) - // Sentence-final punctuation also stays attached. - assert(tokensOf("body", "Hello, world!") == List("Hello,", "world!")) + "split on punctuation and strip it (StandardTokenizer word boundaries)" in { + // `,` is a word boundary, so `"abc,def"` splits into two tokens. + assert(tokensOf("body", "abc,def") == List("abc", "def")) + // Sentence punctuation is a boundary and is dropped, not kept attached. + assert(tokensOf("body", "Hello, world!") == List("Hello", "world")) } // --------------------------------------------------------------------------- diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/keywordSearch/KeywordSearchOpExecSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/keywordSearch/KeywordSearchOpExecSpec.scala index a86ebce4a3..542ccbf4b3 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/keywordSearch/KeywordSearchOpExecSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/keywordSearch/KeywordSearchOpExecSpec.scala @@ -50,7 +50,8 @@ class KeywordSearchOpExecSpec extends AnyFlatSpec with BeforeAndAfter { createTuple("Twitter"), createTuple("안녕하세요"), createTuple("你好"), - createTuple("_!@,-") + createTuple("_!@,-"), + createTuple("everything was absolutely perfect.") ) it should "find exact match with single number" in { @@ -238,4 +239,19 @@ class KeywordSearchOpExecSpec extends AnyFlatSpec with BeforeAndAfter { opExec.close() opDesc.isCaseSensitive = false } + + it should "match a word adjacent to punctuation when case-sensitive" in { + // Regression test: case sensitivity must change only case handling, not + // punctuation -- "perfect" should still match "...perfect." (word-boundary split). + opDesc.attribute = "text" + opDesc.keyword = "perfect" + opDesc.isCaseSensitive = true + val opExec = new KeywordSearchOpExec(objectMapper.writeValueAsString(opDesc)) + opExec.open() + val results = testData.filter(t => opExec.processTuple(t, inputPort).hasNext) + assert(results.length == 1) + assert(results.head.getField[String]("text") == "everything was absolutely perfect.") + opExec.close() + opDesc.isCaseSensitive = false + } }
