This is an automated email from the ASF dual-hosted git repository.
kxiao pushed a commit to branch clucene
in repository https://gitbox.apache.org/repos/asf/doris-thirdparty.git
The following commit(s) were added to refs/heads/clucene by this push:
new d3de160871 [fix](inverted index) special characters cause buffer
overflow in Unicode tokenization. (#210)
d3de160871 is described below
commit d3de160871dc1e2e293e5702e5b870e220ed42e4
Author: zzzxl <[email protected]>
AuthorDate: Tue Apr 30 16:25:32 2024 +0800
[fix](inverted index) special characters cause buffer overflow in Unicode
tokenization. (#210)
---
.../analysis/standard95/StandardTokenizerImpl.cpp | 52 +++++-----------------
src/core/CLucene/util/stringUtil.h | 4 +-
2 files changed, 12 insertions(+), 44 deletions(-)
diff --git a/src/core/CLucene/analysis/standard95/StandardTokenizerImpl.cpp
b/src/core/CLucene/analysis/standard95/StandardTokenizerImpl.cpp
index e8ce8f7768..0dfb116557 100644
--- a/src/core/CLucene/analysis/standard95/StandardTokenizerImpl.cpp
+++ b/src/core/CLucene/analysis/standard95/StandardTokenizerImpl.cpp
@@ -58,7 +58,7 @@ const std::vector<std::string>
StandardTokenizerImpl::ZZ_ERROR_MSG = {
"Error: pushback value was too large"};
StandardTokenizerImpl::StandardTokenizerImpl(lucene::util::Reader* reader)
- : zzBuffer(ZZ_BUFFERSIZE), zzReader(reader) {}
+ : zzReader(reader), zzBuffer((reader == nullptr) ? 0 : reader->size()) {}
std::string_view StandardTokenizerImpl::getText() {
return std::string_view(zzBuffer.data() + zzStartRead,
@@ -67,53 +67,20 @@ std::string_view StandardTokenizerImpl::getText() {
bool StandardTokenizerImpl::zzRefill() {
if (zzStartRead > 0) {
- zzEndRead += zzFinalHighSurrogate;
- zzFinalHighSurrogate = 0;
- std::copy_n(zzBuffer.begin() + zzStartRead, zzEndRead - zzStartRead,
- zzBuffer.begin());
-
- zzEndRead -= zzStartRead;
- zzCurrentPos -= zzStartRead;
- zzMarkedPos -= zzStartRead;
- zzStartRead = 0;
- }
-
- int32_t requested = zzBuffer.size() - zzEndRead - zzFinalHighSurrogate;
- if (requested == 0) {
- return true;
+ return true;
}
- int32_t numRead = zzReader->readCopy(zzBuffer.data(), zzEndRead, requested);
- if (numRead == 0) {
- _CLTHROWA(CL_ERR_Runtime,
- "Reader returned 0 characters. See JFlex examples/zero-reader "
- "for a workaround.");
- }
+ int32_t numRead = zzReader->readCopy(zzBuffer.data(), 0, zzBuffer.size());
if (numRead > 0) {
- zzEndRead += numRead;
-
- int32_t n =
- StringUtil::validate_utf8(std::string_view(zzBuffer.data(),
zzEndRead));
- if (n == -1) {
- yyResetPosition();
- return true;
- }
+ assert(zzBuffer.size() == numRead);
+ zzEndRead += numRead;
- if (n != 0) {
- if (numRead == requested) {
- zzEndRead -= n;
- zzFinalHighSurrogate = n;
- } else {
- int32_t c = zzReader->read();
- if (c == -1) {
+ int32_t n = StringUtil::validate_utf8(std::string_view(zzBuffer.data(),
zzBuffer.size()));
+ if (n != 0) {
return true;
- } else {
- _CLTHROWA(CL_ERR_Runtime, "Why did you come here");
- }
}
- }
- return false;
+ return false;
}
return true;
@@ -126,6 +93,7 @@ void StandardTokenizerImpl::yyclose() {
void StandardTokenizerImpl::yyreset(lucene::util::Reader* reader) {
zzReader = reader;
+ zzBuffer.resize(reader->size());
yyResetPosition();
zzLexicalState = YYINITIAL;
}
@@ -181,7 +149,7 @@ int32_t StandardTokenizerImpl::getNextToken() {
{
while (true) {
- if (zzCurrentPosL < zzEndReadL) {
+ if (zzCurrentPosL < zzEndReadL && (zzCurrentPosL - zzStartRead) <
ZZ_BUFFERSIZE) {
size_t len = 0;
zzInput = decodeUtf8ToCodepoint(
std::string_view(zzBufferL.data() + zzCurrentPosL, zzEndReadL),
diff --git a/src/core/CLucene/util/stringUtil.h
b/src/core/CLucene/util/stringUtil.h
index e7d41e1d83..278c2c4832 100644
--- a/src/core/CLucene/util/stringUtil.h
+++ b/src/core/CLucene/util/stringUtil.h
@@ -297,10 +297,10 @@ public:
} else {
if ((c & 0xC0) != 0x80) return -1;
codepoint = (codepoint << 6) | (c & 0x3F);
- if (!is_valid_codepoint(codepoint)) {
+ bytes_in_char--;
+ if (bytes_in_char == 0 && !is_valid_codepoint(codepoint)) {
return -1;
}
- bytes_in_char--;
surplus_bytes++;
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
