This is an automated email from the ASF dual-hosted git repository.
jianliangqi pushed a commit to branch clucene
in repository https://gitbox.apache.org/repos/asf/doris-thirdparty.git
The following commit(s) were added to refs/heads/clucene by this push:
new 179282c6 [Fix](compile) make PFOR function adapt to compile arch (#116)
179282c6 is described below
commit 179282c60912174c42f7e0884d100bf506d81d0c
Author: airborne12 <[email protected]>
AuthorDate: Mon Sep 4 11:24:06 2023 +0800
[Fix](compile) make PFOR function adapt to compile arch (#116)
---
src/core/CLucene/index/DocumentsWriter.cpp | 10 +---
src/core/CLucene/index/IndexWriter.cpp | 11 +----
src/core/CLucene/index/SDocumentWriter.cpp | 10 +---
src/core/CLucene/index/SegmentTermDocs.cpp | 10 +---
src/core/CLucene/util/PFORUtil.cpp | 75 ++++++++++++++++++++++++++++++
src/core/CLucene/util/PFORUtil.h | 27 +++++++++++
src/core/CMakeLists.txt | 1 +
7 files changed, 108 insertions(+), 36 deletions(-)
diff --git a/src/core/CLucene/index/DocumentsWriter.cpp
b/src/core/CLucene/index/DocumentsWriter.cpp
index 055c33a0..d274f818 100644
--- a/src/core/CLucene/index/DocumentsWriter.cpp
+++ b/src/core/CLucene/index/DocumentsWriter.cpp
@@ -16,6 +16,7 @@
#include "CLucene/util/Array.h"
#include "CLucene/util/CLStreams.h"
#include "CLucene/util/Misc.h"
+#include "CLucene/util/PFORUtil.h"
#include "CLucene/util/_Arrays.h"
#include "CLucene/index/CodeMode.h"
#include "IndexWriter.h"
@@ -31,20 +32,11 @@
#include "_FieldsWriter.h"
#include "_IndexFileNames.h"
#include "_SkipListWriter.h"
-#include "vp4.h"
#include <algorithm>
#include <assert.h>
#include <iostream>
-#if defined(USE_AVX2) && defined(__x86_64__)
-#define P4ENC p4nd1enc256v32
-#define P4NZENC p4nzenc256v32
-#else
-#define P4ENC p4nd1enc128v32
-#define P4NZENC p4nzenc128v32
-#endif
-
CL_NS_USE(util)
CL_NS_USE(store)
CL_NS_USE(analysis)
diff --git a/src/core/CLucene/index/IndexWriter.cpp
b/src/core/CLucene/index/IndexWriter.cpp
index e79d2475..24ac2b08 100644
--- a/src/core/CLucene/index/IndexWriter.cpp
+++ b/src/core/CLucene/index/IndexWriter.cpp
@@ -12,6 +12,7 @@
#include "CLucene/search/Similarity.h"
#include "CLucene/store/Directory.h"
#include "CLucene/util/Misc.h"
+#include "CLucene/util/PFORUtil.h"
#include "IndexReader.h"
#include "IndexWriter.h"
@@ -23,6 +24,7 @@
#include "CLucene/util/Array.h"
#include "CLucene/util/PriorityQueue.h"
#include "CLucene/index/CodeMode.h"
+#include "CLucene/analysis/standard95/StandardAnalyzer.h"
#include "MergePolicy.h"
#include "MergeScheduler.h"
#include "SDocumentWriter.h"
@@ -34,20 +36,11 @@
#include "_SkipListWriter.h"
#include "_Term.h"
#include "_TermInfo.h"
-#include "vp4.h"
#include <algorithm>
#include <memory>
#include <assert.h>
#include <iostream>
-#if defined(USE_AVX2) && defined(__x86_64__)
-#define P4ENC p4nd1enc256v32
-#define P4NZENC p4nzenc256v32
-#else
-#define P4ENC p4nd1enc128v32
-#define P4NZENC p4nzenc128v32
-#endif
-
CL_NS_USE(store)
CL_NS_USE(util)
CL_NS_USE(document)
diff --git a/src/core/CLucene/index/SDocumentWriter.cpp
b/src/core/CLucene/index/SDocumentWriter.cpp
index a91f67d2..5d6de917 100644
--- a/src/core/CLucene/index/SDocumentWriter.cpp
+++ b/src/core/CLucene/index/SDocumentWriter.cpp
@@ -11,6 +11,7 @@
#include "CLucene/util/CLStreams.h"
#include "CLucene/util/Misc.h"
#include "CLucene/util/stringUtil.h"
+#include "CLucene/util/PFORUtil.h"
#include "CLucene/index/CodeMode.h"
#include "_FieldsWriter.h"
@@ -18,20 +19,11 @@
#include "_SkipListWriter.h"
#include "_IndexFileNames.h"
#include "_SegmentMerger.h"
-#include "vp4.h"
#include <algorithm>
#include <vector>
#include <iostream>
-#if defined(USE_AVX2) && defined(__x86_64__)
-#define P4ENC p4nd1enc256v32
-#define P4NZENC p4nzenc256v32
-#else
-#define P4ENC p4nd1enc128v32
-#define P4NZENC p4nzenc128v32
-#endif
-
CL_NS_USE(util)
CL_NS_USE(store)
CL_NS_USE(analysis)
diff --git a/src/core/CLucene/index/SegmentTermDocs.cpp
b/src/core/CLucene/index/SegmentTermDocs.cpp
index bc537f47..765f6424 100644
--- a/src/core/CLucene/index/SegmentTermDocs.cpp
+++ b/src/core/CLucene/index/SegmentTermDocs.cpp
@@ -9,21 +9,13 @@
#include "CLucene/store/IndexInput.h"
#include "CLucene/index/CodeMode.h"
+#include "CLucene/util/PFORUtil.h"
#include "Term.h"
-#include "vp4.h"
#include <assert.h>
#include <memory>
#include <iostream>
-#if defined(USE_AVX2) && defined(__x86_64__)
-#define P4DEC p4nd1dec256v32
-#define P4NZDEC p4nzdec256v32
-#else
-#define P4DEC p4nd1dec128v32
-#define P4NZDEC p4nzdec128v32
-#endif
-
CL_NS_DEF(index)
SegmentTermDocs::SegmentTermDocs(const SegmentReader *_parent) :
parent(_parent), freqStream(_parent->freqStream->clone()),
diff --git a/src/core/CLucene/util/PFORUtil.cpp
b/src/core/CLucene/util/PFORUtil.cpp
new file mode 100644
index 00000000..eaf628c7
--- /dev/null
+++ b/src/core/CLucene/util/PFORUtil.cpp
@@ -0,0 +1,75 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+#include "PFORUtil.h"
+#include "vp4.h"
+
+namespace {
+using DEC_FUNC = size_t (*)(unsigned char *__restrict, size_t, uint32_t
*__restrict);
+using ENC_FUNC = size_t (*)(uint32_t *__restrict in, size_t n, unsigned char
*__restrict out);
+DEC_FUNC g_p4nd1dec;
+DEC_FUNC g_p4nzdec;
+ENC_FUNC g_p4nd1enc;
+ENC_FUNC g_p4nzenc;
+} // anonymous namespace
+
+__attribute__((constructor)) void SelectPFORFunctions() {
+ uint32_t eax, ebx, ecx, edx;
+ __cpuid(1, eax, ebx, ecx, edx);
+
+ bool sse2 = (edx & bit_SSE2) != 0;
+ bool sse42 = (ecx & bit_SSE4_2) != 0;
+#if (defined(__i386) || defined(__x86_64__))
+#if defined(USE_AVX2)
+ g_p4nd1dec = p4nd1dec256v32;
+ g_p4nzdec = p4nzdec256v32;
+ g_p4nd1enc = p4nd1enc256v32;
+ g_p4nzenc = p4nzenc256v32;
+#else
+ if (sse42) {
+ g_p4nd1dec = p4nd1dec128v32;
+ g_p4nzdec = p4nzdec128v32;
+ g_p4nd1enc = p4nd1enc128v32;
+ g_p4nzenc = p4nzenc128v32;
+ } else {
+ g_p4nd1dec = p4nd1dec32;
+ g_p4nzdec = p4nzdec32;
+ g_p4nd1enc = p4nd1enc32;
+ g_p4nzenc = p4nzenc32;
+ }
+#endif
+#else
+ g_p4nd1dec = p4nd1dec32;
+ g_p4nzdec = p4nzdec32;
+ g_p4nd1enc = p4nd1enc32;
+ g_p4nzenc = p4nzenc32;
+#endif
+}
+
+size_t P4DEC(unsigned char *__restrict in, size_t n, uint32_t *__restrict out)
{
+ return g_p4nd1dec(in, n, out);
+}
+size_t P4NZDEC(unsigned char *__restrict in, size_t n, uint32_t *__restrict
out) {
+ return g_p4nzdec(in, n, out);
+}
+
+size_t P4ENC(uint32_t *__restrict in, size_t n, unsigned char *__restrict out)
{
+ return g_p4nd1enc(in, n, out);
+}
+
+size_t P4NZENC(uint32_t *__restrict in, size_t n, unsigned char *__restrict
out) {
+ return g_p4nzenc(in, n, out);
+}
diff --git a/src/core/CLucene/util/PFORUtil.h b/src/core/CLucene/util/PFORUtil.h
new file mode 100644
index 00000000..5ba664aa
--- /dev/null
+++ b/src/core/CLucene/util/PFORUtil.h
@@ -0,0 +1,27 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+#pragma once
+
+#include <cstddef>
+#include <cstdint>
+#include <cpuid.h>
+
+size_t P4DEC(unsigned char *__restrict in, size_t n, uint32_t *__restrict out);
+size_t P4NZDEC(unsigned char *__restrict in, size_t n, uint32_t *__restrict
out);
+size_t P4ENC(uint32_t *__restrict in, size_t n, unsigned char *__restrict out);
+size_t P4NZENC(uint32_t *__restrict in, size_t n, unsigned char *__restrict
out);
+
diff --git a/src/core/CMakeLists.txt b/src/core/CMakeLists.txt
index d9269e9f..e2264120 100644
--- a/src/core/CMakeLists.txt
+++ b/src/core/CMakeLists.txt
@@ -55,6 +55,7 @@ SET(clucene_core_Files
./CLucene/util/FutureArrays.cpp
./CLucene/util/NumericUtils.cpp
./CLucene/util/stringUtil.cpp
+ ./CLucene/util/PFORUtil.cpp
./CLucene/queryParser/FastCharStream.cpp
./CLucene/queryParser/MultiFieldQueryParser.cpp
./CLucene/queryParser/QueryParser.cpp
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
