This is an automated email from the ASF dual-hosted git repository.
airborne pushed a commit to branch clucene
in repository https://gitbox.apache.org/repos/asf/doris-thirdparty.git
The following commit(s) were added to refs/heads/clucene by this push:
new 63d82e79206 [opt](index writer) allow non analyzer for non-tokenize
flag (#342)
63d82e79206 is described below
commit 63d82e79206ad7da77b9d4e730044b37c4ef492b
Author: airborne12 <[email protected]>
AuthorDate: Tue Jul 29 00:04:48 2025 +0800
[opt](index writer) allow non analyzer for non-tokenize flag (#342)
---
src/core/CLucene/index/IndexWriter.cpp | 2 +-
src/core/CLucene/index/SDocumentWriter.cpp | 7 +++----
2 files changed, 4 insertions(+), 5 deletions(-)
diff --git a/src/core/CLucene/index/IndexWriter.cpp
b/src/core/CLucene/index/IndexWriter.cpp
index d6d0a28d5dd..7ec6f56817d 100644
--- a/src/core/CLucene/index/IndexWriter.cpp
+++ b/src/core/CLucene/index/IndexWriter.cpp
@@ -298,7 +298,7 @@ void IndexWriter::init(Directory *d, Analyzer *a, const
bool create, const bool
_CLTHROWA(CL_ERR_IllegalArgument, "IndexWriter::init: Only
support SDocumentsWriter");
}
} else {
- _CLTHROWA(CL_ERR_IllegalArgument, "IndexWriter::init: Only support
SDocumentsWriter");
+ docWriter = _CLNEW SDocumentsWriter<char>(directory, this);
}
pushMaxBufferedDocs();
diff --git a/src/core/CLucene/index/SDocumentWriter.cpp
b/src/core/CLucene/index/SDocumentWriter.cpp
index 637754bdb9f..b41290741be 100644
--- a/src/core/CLucene/index/SDocumentWriter.cpp
+++ b/src/core/CLucene/index/SDocumentWriter.cpp
@@ -632,10 +632,6 @@ void
SDocumentsWriter<T>::ThreadState::FieldData::addPosition(Token *token) {
template<typename T>
void SDocumentsWriter<T>::ThreadState::FieldData::invertField(Field *field,
Analyzer *sanalyzer, const int32_t maxFieldLength) {
-
- if (length > 0)
- position += sanalyzer->getPositionIncrementGap(fieldInfo->name);
-
if (!field->isTokenized()) {// un-tokenized field
const T *stringValue = (T *) field->rawStringValue();
const size_t valueLength = field->getFieldDataLength();
@@ -649,6 +645,9 @@ void
SDocumentsWriter<T>::ThreadState::FieldData::invertField(Field *field, Anal
offset += valueLength;
length++;
} else {// tokenized field
+ if (length > 0) {
+ position += sanalyzer->getPositionIncrementGap(fieldInfo->name);
+ }
TokenStream *stream;
TokenStream *streamValue = field->tokenStreamValue();
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
