This is an automated email from the ASF dual-hosted git repository.
airborne pushed a commit to branch clucene-3.1
in repository https://gitbox.apache.org/repos/asf/doris-thirdparty.git
The following commit(s) were added to refs/heads/clucene-3.1 by this push:
new aa61e9fb0ec branch-3.1: [opt](index writer) allow non analyzer for
non-tokenize flag (#342) (#349)
aa61e9fb0ec is described below
commit aa61e9fb0ec90dc5a52834d677322266dcabe38d
Author: Jack <[email protected]>
AuthorDate: Wed Aug 13 11:45:28 2025 +0800
branch-3.1: [opt](index writer) allow non analyzer for non-tokenize flag
(#342) (#349)
---
src/core/CLucene/index/IndexWriter.cpp | 2 +-
src/core/CLucene/index/SDocumentWriter.cpp | 7 +++----
2 files changed, 4 insertions(+), 5 deletions(-)
diff --git a/src/core/CLucene/index/IndexWriter.cpp
b/src/core/CLucene/index/IndexWriter.cpp
index 6a586e3e7bd..6d3bab6905f 100644
--- a/src/core/CLucene/index/IndexWriter.cpp
+++ b/src/core/CLucene/index/IndexWriter.cpp
@@ -298,7 +298,7 @@ void IndexWriter::init(Directory *d, Analyzer *a, const
bool create, const bool
_CLTHROWA(CL_ERR_IllegalArgument, "IndexWriter::init: Only
support SDocumentsWriter");
}
} else {
- _CLTHROWA(CL_ERR_IllegalArgument, "IndexWriter::init: Only support
SDocumentsWriter");
+ docWriter = _CLNEW SDocumentsWriter<char>(directory, this);
}
pushMaxBufferedDocs();
diff --git a/src/core/CLucene/index/SDocumentWriter.cpp
b/src/core/CLucene/index/SDocumentWriter.cpp
index 637754bdb9f..b41290741be 100644
--- a/src/core/CLucene/index/SDocumentWriter.cpp
+++ b/src/core/CLucene/index/SDocumentWriter.cpp
@@ -632,10 +632,6 @@ void
SDocumentsWriter<T>::ThreadState::FieldData::addPosition(Token *token) {
template<typename T>
void SDocumentsWriter<T>::ThreadState::FieldData::invertField(Field *field,
Analyzer *sanalyzer, const int32_t maxFieldLength) {
-
- if (length > 0)
- position += sanalyzer->getPositionIncrementGap(fieldInfo->name);
-
if (!field->isTokenized()) {// un-tokenized field
const T *stringValue = (T *) field->rawStringValue();
const size_t valueLength = field->getFieldDataLength();
@@ -649,6 +645,9 @@ void
SDocumentsWriter<T>::ThreadState::FieldData::invertField(Field *field, Anal
offset += valueLength;
length++;
} else {// tokenized field
+ if (length > 0) {
+ position += sanalyzer->getPositionIncrementGap(fieldInfo->name);
+ }
TokenStream *stream;
TokenStream *streamValue = field->tokenStreamValue();
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
