This is an automated email from the ASF dual-hosted git repository.
jianliangqi pushed a commit to branch clucene
in repository https://gitbox.apache.org/repos/asf/doris-thirdparty.git
The following commit(s) were added to refs/heads/clucene by this push:
new 97d40fdf [feature](invert index) add the reader buffer as a parameter
(#137)
97d40fdf is described below
commit 97d40fdf9477580672060c5684bc5ec09fb97bd3
Author: zzzxl <[email protected]>
AuthorDate: Tue Nov 21 14:06:12 2023 +0800
[feature](invert index) add the reader buffer as a parameter (#137)
---
src/core/CLucene/index/DirectoryIndexReader.cpp | 20 ++++++++++++++++++--
src/core/CLucene/index/DirectoryIndexReader.h | 4 ++++
src/core/CLucene/index/IndexReader.cpp | 5 +++++
src/core/CLucene/index/IndexReader.h | 3 +++
src/core/CLucene/index/MultiSegmentReader.cpp | 4 ++--
src/core/CLucene/index/SegmentReader.cpp | 5 +++++
src/core/CLucene/index/_MultiSegmentReader.h | 3 ++-
src/core/CLucene/index/_SegmentHeader.h | 2 ++
8 files changed, 41 insertions(+), 5 deletions(-)
diff --git a/src/core/CLucene/index/DirectoryIndexReader.cpp
b/src/core/CLucene/index/DirectoryIndexReader.cpp
index 15020eea..d3acb6f1 100644
--- a/src/core/CLucene/index/DirectoryIndexReader.cpp
+++ b/src/core/CLucene/index/DirectoryIndexReader.cpp
@@ -4,6 +4,7 @@
* Distributable under the terms of either the Apache License (Version 2.0) or
* the GNU Lesser General Public License, as specified in the COPYING file.
------------------------------------------------------------------------------*/
+#include <cstdint>
#include "CLucene/_ApiHeader.h"
#include "DirectoryIndexReader.h"
#include "_IndexFileDeleter.h"
@@ -141,6 +142,8 @@ CL_NS_DEF(index)
class DirectoryIndexReader::FindSegmentsFile_Open: public
SegmentInfos::FindSegmentsFile<DirectoryIndexReader*>{
bool closeDirectory;
IndexDeletionPolicy* deletionPolicy;
+ int32_t readBufferSize_ = -1;
+
protected:
DirectoryIndexReader* doBody(const char* segmentFileName) {
@@ -150,9 +153,9 @@ CL_NS_DEF(index)
DirectoryIndexReader* reader;
if (infos->size() == 1) { // index is optimized
- reader = SegmentReader::get(infos, infos->info(0), closeDirectory);
+ reader = SegmentReader::get(infos, infos->info(0), readBufferSize_,
closeDirectory);
} else {
- reader = _CLNEW MultiSegmentReader(directory, infos, closeDirectory);
+ reader = _CLNEW MultiSegmentReader(directory, infos, closeDirectory,
readBufferSize_);
}
reader->setDeletionPolicy(deletionPolicy);
return reader;
@@ -165,6 +168,12 @@ CL_NS_DEF(index)
this->closeDirectory = closeDirectory;
this->deletionPolicy = deletionPolicy;
}
+
+ FindSegmentsFile_Open(bool closeDirectory, int32_t readBufferSize,
+ IndexDeletionPolicy* deletionPolicy,
CL_NS(store)::Directory* dir)
+ : FindSegmentsFile_Open(closeDirectory, deletionPolicy, dir) {
+ readBufferSize_ = readBufferSize;
+ }
};
DirectoryIndexReader* DirectoryIndexReader::open(Directory* __directory,
bool closeDirectory, IndexDeletionPolicy* deletionPolicy) {
@@ -172,6 +181,13 @@ CL_NS_DEF(index)
return runner.run();
}
+ DirectoryIndexReader* DirectoryIndexReader::open(Directory* __directory,
int32_t readBufferSize,
+ bool closeDirectory,
+ IndexDeletionPolicy*
deletionPolicy) {
+ DirectoryIndexReader::FindSegmentsFile_Open runner(closeDirectory,
readBufferSize,
+ deletionPolicy,
__directory);
+ return runner.run();
+ }
class DirectoryIndexReader::FindSegmentsFile_Reopen: public
SegmentInfos::FindSegmentsFile<DirectoryIndexReader*>{
bool closeDirectory;
diff --git a/src/core/CLucene/index/DirectoryIndexReader.h
b/src/core/CLucene/index/DirectoryIndexReader.h
index 62d31508..e614f3d3 100644
--- a/src/core/CLucene/index/DirectoryIndexReader.h
+++ b/src/core/CLucene/index/DirectoryIndexReader.h
@@ -7,6 +7,7 @@
#ifndef _lucene_index_DirectoryIndexReader_
#define _lucene_index_DirectoryIndexReader_
+#include <cstdint>
#include "IndexReader.h"
CL_CLASS_DEF(store,LuceneLock)
@@ -84,6 +85,9 @@ public:
CLUCENE_LOCAL_DECL DirectoryIndexReader(CL_NS(store)::Directory* directory,
SegmentInfos* segmentInfos, bool closeDirectory);
CLUCENE_LOCAL_DECL static DirectoryIndexReader*
open(CL_NS(store)::Directory* directory, bool closeDirectory,
IndexDeletionPolicy* deletionPolicy);
+ CLUCENE_LOCAL_DECL static DirectoryIndexReader*
open(CL_NS(store)::Directory* directory,
+ int32_t readBufferSize,
bool closeDirectory,
+ IndexDeletionPolicy*
deletionPolicy);
IndexReader* reopen();
diff --git a/src/core/CLucene/index/IndexReader.cpp
b/src/core/CLucene/index/IndexReader.cpp
index 0fb2fe59..a63b8d1a 100644
--- a/src/core/CLucene/index/IndexReader.cpp
+++ b/src/core/CLucene/index/IndexReader.cpp
@@ -126,6 +126,11 @@ CL_NS_DEF(index)
return DirectoryIndexReader::open(directory, closeDirectory,
deletionPolicy);
}
+ IndexReader* IndexReader::open(Directory* directory, int32_t readBufferSize,
bool closeDirectory,
+ IndexDeletionPolicy* deletionPolicy) {
+ return DirectoryIndexReader::open(directory, readBufferSize,
closeDirectory, deletionPolicy);
+ }
+
IndexReader* IndexReader::reopen(){
_CLTHROWA(CL_ERR_UnsupportedOperation, "This reader does not support
reopen().");
}
diff --git a/src/core/CLucene/index/IndexReader.h
b/src/core/CLucene/index/IndexReader.h
index defb6e75..59e937df 100644
--- a/src/core/CLucene/index/IndexReader.h
+++ b/src/core/CLucene/index/IndexReader.h
@@ -156,6 +156,9 @@ public:
*/
static IndexReader* open(CL_NS(store)::Directory* directory, bool
closeDirectoryOnCleanup=false, IndexDeletionPolicy* deletionPolicy=NULL);
+ static IndexReader* open(CL_NS(store)::Directory* directory, int32_t
readBufferSize,
+ bool closeDirectoryOnCleanup = false,
+ IndexDeletionPolicy* deletionPolicy = NULL);
/**
* Refreshes an IndexReader if the index has changed since this instance
diff --git a/src/core/CLucene/index/MultiSegmentReader.cpp
b/src/core/CLucene/index/MultiSegmentReader.cpp
index ef006c20..495acaa4 100644
--- a/src/core/CLucene/index/MultiSegmentReader.cpp
+++ b/src/core/CLucene/index/MultiSegmentReader.cpp
@@ -45,7 +45,7 @@ void
MultiSegmentReader::initialize(CL_NS(util)::ArrayBase<IndexReader*>* _subRe
starts[subReaders->length] = _maxDoc;
}
-MultiSegmentReader::MultiSegmentReader(CL_NS(store)::Directory* directory,
SegmentInfos* sis, bool closeDirectory):
+MultiSegmentReader::MultiSegmentReader(CL_NS(store)::Directory* directory,
SegmentInfos* sis, bool closeDirectory, int32_t readBufferSize):
DirectoryIndexReader(directory,sis,closeDirectory),
normsCache(NormsCacheType(true,true))
{
@@ -57,7 +57,7 @@
MultiSegmentReader::MultiSegmentReader(CL_NS(store)::Directory* directory, Segme
ArrayBase<IndexReader*>* readers = _CLNEW
ObjectArray<IndexReader>(sis->size());
for (int32_t i = (int32_t)sis->size()-1; i >= 0; i--) {
try {
- readers->values[i] = SegmentReader::get(sis->info(i));
+ readers->values[i] = SegmentReader::get(sis->info(i), readBufferSize);
} catch(CLuceneError& err) {
if ( err.number() != CL_ERR_IO ) throw err;
diff --git a/src/core/CLucene/index/SegmentReader.cpp
b/src/core/CLucene/index/SegmentReader.cpp
index e0ec0c14..f7741a9f 100644
--- a/src/core/CLucene/index/SegmentReader.cpp
+++ b/src/core/CLucene/index/SegmentReader.cpp
@@ -239,6 +239,11 @@ SegmentReader *SegmentReader::get(SegmentInfos *sis,
SegmentInfo *si,
bool closeDir) {
return get(si->dir, si, sis, closeDir, true,
BufferedIndexInput::BUFFER_SIZE, false);
}
+
+SegmentReader *SegmentReader::get(SegmentInfos *sis, SegmentInfo *si, int32_t
readBufferSize, bool closeDir) {
+ return get(si->dir, si, sis, closeDir, true, readBufferSize, false);
+}
+
/**
* @throws CorruptIndexException if the index is corrupt
* @throws IOException if there is a low-level IO error
diff --git a/src/core/CLucene/index/_MultiSegmentReader.h
b/src/core/CLucene/index/_MultiSegmentReader.h
index 46b32caa..52f700d1 100644
--- a/src/core/CLucene/index/_MultiSegmentReader.h
+++ b/src/core/CLucene/index/_MultiSegmentReader.h
@@ -8,6 +8,7 @@
#define _lucene_index_MultiSegmentReader
+#include <cstdint>
#include "DirectoryIndexReader.h"
#include "IndexReader.h"
CL_CLASS_DEF(document,Document)
@@ -58,7 +59,7 @@ protected:
public:
/** Construct reading the named set of readers. */
- MultiSegmentReader(CL_NS(store)::Directory* directory, SegmentInfos* sis,
bool closeDirectory);
+ MultiSegmentReader(CL_NS(store)::Directory* directory, SegmentInfos* sis,
bool closeDirectory, int32_t readBufferSize = -1);
/** This contructor is only used for {@link #reopen()} */
CLUCENE_LOCAL_DECL MultiSegmentReader(
diff --git a/src/core/CLucene/index/_SegmentHeader.h
b/src/core/CLucene/index/_SegmentHeader.h
index e177f576..bf988a2f 100644
--- a/src/core/CLucene/index/_SegmentHeader.h
+++ b/src/core/CLucene/index/_SegmentHeader.h
@@ -367,6 +367,8 @@ public:
*/
static SegmentReader* get(SegmentInfos* sis, SegmentInfo* si, bool closeDir);
+ static SegmentReader *get(SegmentInfos *sis, SegmentInfo *si, int32_t
readBufferSize, bool closeDir);
+
/**
* @throws CorruptIndexException if the index is corrupt
* @throws IOException if there is a low-level IO error
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
