Mgautierfr has uploaded a new change for review.
https://gerrit.wikimedia.org/r/295520
Change subject: Add a indexer.
......................................................................
Add a indexer.
This indexer is not used.
This is mainly code from kiwix-indexer imported in openzim.
Unused function in *Tools has been removed.
No dependency to xapian.
Change-Id: I0683706a136fb2303234e4caee77e9221714a5b1
---
M zimwriterfs/Makefile.am
M zimwriterfs/configure.ac
A zimwriterfs/indexer.cpp
A zimwriterfs/indexer.h
A zimwriterfs/pathTools.cpp
A zimwriterfs/pathTools.h
A zimwriterfs/resourceTools.cpp
A zimwriterfs/resourceTools.h
M zimwriterfs/tools.cpp
M zimwriterfs/tools.h
10 files changed, 1,009 insertions(+), 1 deletion(-)
git pull ssh://gerrit.wikimedia.org:29418/openzim refs/changes/20/295520/1
diff --git a/zimwriterfs/Makefile.am b/zimwriterfs/Makefile.am
index 6e46553..e54c64d 100644
--- a/zimwriterfs/Makefile.am
+++ b/zimwriterfs/Makefile.am
@@ -5,4 +5,10 @@
zimwriterfs.cpp \
tools.cpp \
article.cpp \
- articlesource.cpp
+ articlesource.cpp \
+ indexer.cpp \
+ resourceTools.cpp \
+ pathTools.cpp
+
+zimwriterfs_CXXFLAGS = $(ICU_CFLAGS)
+zimwriterfs_LDFLAGS = $(ICU_LDFLAGS)
diff --git a/zimwriterfs/configure.ac b/zimwriterfs/configure.ac
index fb12c8f..ba23cd9 100644
--- a/zimwriterfs/configure.ac
+++ b/zimwriterfs/configure.ac
@@ -71,6 +71,81 @@
AC_DEFINE_UNQUOTED(LZMA_MEMORY_SIZE, 128, [set lzma uncompress memory size to
number of MB])
AC_DEFINE(ENABLE_LZMA, [1], [defined if lzma compression is enabled])
+
+function findLibrary {
+ found=0
+ for f in $(echo $LIBS_ROOT|tr ":" "\n") ; do
+ sf=`find $f -name $1 | grep $ARCH | head -1 2> /dev/null`
+ if [[ -f "$sf" -a $found -eq 0 ]]
+ then
+ found=1
+ echo $sf
+ fi
+ done
+ if [[ $found -eq 0 ]]
+ then
+ for f in $(echo $LIBS_ROOT|tr ":" "\n") ; do
+ sf=`find $f -name $1 | head -1 2> /dev/null`
+ if [[ -f "$sf" -a $found -eq 0 ]]
+ then
+ found=1
+ echo $sf
+ fi
+ done
+ fi
+ if [[ $found -eq 0 ]]
+ then
+ echo "no"
+ fi
+}
+
+
+####################################################
+############ ICU
+####################################################
+
+
+ICU_CFLAGS=""
+ICU_LDFLAGS="-licui18n -licuuc -licudata" # replaced by icu-config
+ICU_STATIC_LDFLAGS=""
+
+# if --with-x, add path to LIBRARY_PATH
+AC_ARG_WITH(icu,
+ AC_HELP_STRING([--with-icu=DIR], [alternate location for
icu-config]),
+ export
LIBRARY_PATH="${withval}:${LIBRARY_PATH}";ICU_PATH=${withval}
+ )
+
+# look for shared library.
+# AC_CHECK_HEADER([zlib.h],, [AC_MSG_ERROR([[cannot find zlib header]])])
+# AC_CHECK_LIB([z], [zlibVersion],, [AC_MSG_ERROR([[cannot find
zlib]]);COMPILE_ICU=1])
+# ICU_FILES=`findLibrary "libicuuc.${SHARED_EXT}"`
+
+AC_CHECK_TOOL(HAVE_ICU_CONFIG, icu-config,, "${ICU_PATH}:${PATH}")
+if test [ ! "$HAVE_ICU_CONFIG" ]
+then
+ AC_MSG_ERROR([[cannot find icu-config]])
+else
+ OLDPATH=$PATH
+ PATH="${ICU_PATH}:${PATH}"
+ ICU_CFLAGS=`icu-config --cxxflags`;
+ ICU_LDFLAGS=`icu-config --ldflags`;
+ ICU_VER=`icu-config --version`;
+ ICU_FILES="`findLibrary "libicuuc.${SHARED_EXT}"` `findLibrary
"libicudata.${SHARED_EXT}"` `findLibrary "libicui18n.${SHARED_EXT}"`"
+ PATH=$OLDPATH
+ if [[ $ICU_VER \< "4.2" ]]
+ then
+ AC_MSG_ERROR([[You need a version of libicu >= 4.2]])
+ fi
+fi
+
+
+AC_SUBST(ICU_CFLAGS)
+AC_SUBST(ICU_LDFLAGS)
+AC_SUBST(ICU_STATIC_LDFLAGS)
+AC_SUBST(ICU_FILES)
+AC_SUBST(COMPILED_ICUDATA_DAT)
+
+
# Configure the output files
AC_CONFIG_FILES([
Makefile
diff --git a/zimwriterfs/indexer.cpp b/zimwriterfs/indexer.cpp
new file mode 100644
index 0000000..7820a32
--- /dev/null
+++ b/zimwriterfs/indexer.cpp
@@ -0,0 +1,262 @@
+/*
+ * Copyright 2011-2014 Emmanuel Engelhart <[email protected]>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ */
+
+#include "indexer.h"
+#include "resourceTools.h"
+#include "pathTools.h"
+#include <unistd.h>
+
+ /* Count word */
+ unsigned int Indexer::countWords(const string &text) {
+ unsigned int numWords = 1;
+ unsigned int length = text.size();
+
+ for(unsigned int i=0; i<length;) {
+ while(i<length && text[i] != ' ') {
+ i++;
+ }
+ numWords++;
+ i++;
+ }
+
+ return numWords;
+ }
+
+ /* Constructor */
+ Indexer::Indexer() :
+ keywordsBoostFactor(3),
+ verboseFlag(false) {
+
+ /* Initialize mutex */
+ pthread_mutex_init(&threadIdsMutex, NULL);
+ pthread_mutex_init(&toIndexQueueMutex, NULL);
+ pthread_mutex_init(&articleIndexerRunningMutex, NULL);
+ pthread_mutex_init(&articleCountMutex, NULL);
+ pthread_mutex_init(&zimIdMutex, NULL);
+ pthread_mutex_init(&indexPathMutex, NULL);
+ pthread_mutex_init(&verboseMutex, NULL);
+ }
+
+ /* Destructor */
+ Indexer::~Indexer() {
+ }
+
+ /* Read the stopwords */
+ void Indexer::readStopWords(const string languageCode) {
+ std::string stopWord;
+ std::istringstream file(getResourceAsString("stopwords/" + languageCode));
+
+ this->stopWords.clear();
+
+ while (getline(file, stopWord, '\n')) {
+ this->stopWords.push_back(stopWord);
+ }
+ }
+
+ /* Article indexer methods */
+ void *Indexer::indexArticles(void *ptr) {
+ pthread_setcanceltype(PTHREAD_CANCEL_DEFERRED, NULL);
+ Indexer *self = (Indexer *)ptr;
+ unsigned int indexedArticleCount = 0;
+ indexerToken token;
+
+ self->indexingPrelude(self->getIndexPath());
+
+ while (self->popFromToIndexQueue(token)) {
+ self->index(token.url,
+ token.accentedTitle,
+ token.title,
+ token.keywords,
+ token.content,
+ token.snippet,
+ token.size,
+ token.wordCount
+ );
+
+ indexedArticleCount += 1;
+
+ /* Make a hard-disk flush every 10.000 articles */
+ if (indexedArticleCount % 5000 == 0) {
+ self->flush();
+ }
+
+ /* Test if the thread should be cancelled */
+ pthread_testcancel();
+ }
+ self->indexingPostlude();
+
+ /* Write content id file */
+ string path = appendToDirectory(self->getIndexPath(), "content.id");
+ writeTextFile(path, self->getZimId());
+
+ usleep(100);
+
+ self->articleIndexerRunning(false);
+ pthread_exit(NULL);
+ return NULL;
+ }
+
+ void Indexer::articleIndexerRunning(bool value) {
+ pthread_mutex_lock(&articleIndexerRunningMutex);
+ this->articleIndexerRunningFlag = value;
+ pthread_mutex_unlock(&articleIndexerRunningMutex);
+ }
+
+ bool Indexer::isArticleIndexerRunning() {
+ pthread_mutex_lock(&articleIndexerRunningMutex);
+ bool retVal = this->articleIndexerRunningFlag;
+ pthread_mutex_unlock(&articleIndexerRunningMutex);
+ return retVal;
+ }
+
+ /* ToIndexQueue methods */
+ bool Indexer::isToIndexQueueEmpty() {
+ pthread_mutex_lock(&toIndexQueueMutex);
+ bool retVal = this->toIndexQueue.empty();
+ pthread_mutex_unlock(&toIndexQueueMutex);
+ return retVal;
+ }
+
+ void Indexer::pushToIndexQueue(indexerToken &token) {
+ pthread_mutex_lock(&toIndexQueueMutex);
+ this->toIndexQueue.push(token);
+ pthread_mutex_unlock(&toIndexQueueMutex);
+ usleep(int(this->toIndexQueue.size() / 200) / 10 * 1000);
+ }
+
+ bool Indexer::popFromToIndexQueue(indexerToken &token) {
+ while (this->isToIndexQueueEmpty()) {
+ usleep(500);
+ if (this->getVerboseFlag()) {
+ std::cout << "Waiting... ToIndexQueue is empty for now..." << std::endl;
+ }
+
+ pthread_testcancel();
+ }
+
+ pthread_mutex_lock(&toIndexQueueMutex);
+ token = this->toIndexQueue.front();
+ this->toIndexQueue.pop();
+ pthread_mutex_unlock(&toIndexQueueMutex);
+
+ if (token.title == ""){
+ //This is a empty token, end of the queue.
+ return false;
+ }
+ return true;
+ }
+
+ /* Index methods */
+ void Indexer::setIndexPath(const string path) {
+ pthread_mutex_lock(&indexPathMutex);
+ this->indexPath = path;
+ pthread_mutex_unlock(&indexPathMutex);
+ }
+
+ string Indexer::getIndexPath() {
+ pthread_mutex_lock(&indexPathMutex);
+ string retVal = this->indexPath;
+ pthread_mutex_unlock(&indexPathMutex);
+ return retVal;
+ }
+
+ void Indexer::setArticleCount(const unsigned int articleCount) {
+ pthread_mutex_lock(&articleCountMutex);
+ this->articleCount = articleCount;
+ pthread_mutex_unlock(&articleCountMutex);
+ }
+
+ unsigned int Indexer::getArticleCount() {
+ pthread_mutex_lock(&articleCountMutex);
+ unsigned int retVal = this->articleCount;
+ pthread_mutex_unlock(&articleCountMutex);
+ return retVal;
+ }
+
+ void Indexer::setZimId(const string id) {
+ pthread_mutex_lock(&zimIdMutex);
+ this->zimId = id;
+ pthread_mutex_unlock(&zimIdMutex);
+ }
+
+ string Indexer::getZimId() {
+ pthread_mutex_lock(&zimIdMutex);
+ string retVal = this->zimId;
+ pthread_mutex_unlock(&zimIdMutex);
+ return retVal;
+ }
+
+ /* Manage */
+ bool Indexer::start(const string indexPath) {
+ if (this->getVerboseFlag()) {
+ std::cout << "Indexing starting..." <<std::endl;
+ }
+
+ this->setArticleCount(0);
+ this->setIndexPath(indexPath);
+
+ pthread_mutex_lock(&threadIdsMutex);
+
+ this->articleIndexerRunning(true);
+ pthread_create(&(this->articleIndexer), NULL, Indexer::indexArticles,
(void*)this);
+ pthread_detach(this->articleIndexer);
+ pthread_mutex_unlock(&threadIdsMutex);
+
+ return true;
+ }
+
+ bool Indexer::isRunning() {
+ if (this->getVerboseFlag()) {
+ std::cout << "isArticleIndexer running: " <<
(this->isArticleIndexerRunning() ? "yes" : "no") << std::endl;
+ }
+
+ return this->isArticleIndexerRunning();
+ }
+
+ bool Indexer::stop() {
+ if (this->isRunning()) {
+ bool isArticleIndexerRunning = this->isArticleIndexerRunning();
+
+ pthread_mutex_lock(&threadIdsMutex);
+
+ if (isArticleIndexerRunning) {
+ pthread_cancel(this->articleIndexer);
+ this->articleIndexerRunning(false);
+ }
+
+ pthread_mutex_unlock(&threadIdsMutex);
+ }
+
+ return true;
+ }
+
+ /* Manage the verboseFlag */
+ void Indexer::setVerboseFlag(const bool value) {
+ pthread_mutex_lock(&verboseMutex);
+ this->verboseFlag = value;
+ pthread_mutex_unlock(&verboseMutex);
+ }
+
+ bool Indexer::getVerboseFlag() {
+ bool value;
+ pthread_mutex_lock(&verboseMutex);
+ value = this->verboseFlag;
+ pthread_mutex_unlock(&verboseMutex);
+ return value;
+ }
diff --git a/zimwriterfs/indexer.h b/zimwriterfs/indexer.h
new file mode 100644
index 0000000..02d989b
--- /dev/null
+++ b/zimwriterfs/indexer.h
@@ -0,0 +1,134 @@
+/*
+ * Copyright 2014 Emmanuel Engelhart <[email protected]>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ */
+
+#ifndef OPENZIM_ZIMWRITERFS_INDEXER_H
+#define OPENZIM_ZIMWRITERFS_INDEXER_H
+
+#include <string>
+#include <vector>
+#include <stack>
+#include <queue>
+#include <fstream>
+#include <iostream>
+#include <sstream>
+
+#include <pthread.h>
+/*#include <stringTools.h>
+#include <otherTools.h>
+#include <resourceTools.h>*/
+#include <zim/file.h>
+#include <zim/article.h>
+#include <zim/fileiterator.h>
+#include <zim/writer/zimcreator.h>
+/*#include "reader.h"*/
+
+using namespace std;
+
+struct indexerToken {
+ string url;
+ string accentedTitle;
+ string title;
+ string keywords;
+ string content;
+ string snippet;
+ string size;
+ string wordCount;
+};
+
+class Indexer {
+
+ public:
+ Indexer();
+ virtual ~Indexer();
+
+ bool start(const string indexPath);
+ bool stop();
+ bool isRunning();
+ void setVerboseFlag(const bool value);
+ void pushToIndexQueue(indexerToken &token);
+
+ protected:
+ virtual void indexingPrelude(const string indexPath) = 0;
+ virtual void index(const string &url,
+ const string &title,
+ const string &unaccentedTitle,
+ const string &keywords,
+ const string &content,
+ const string &snippet,
+ const string &size,
+ const string &wordCount) = 0;
+ virtual void flush() = 0;
+ virtual void indexingPostlude() = 0;
+
+ /* Stop words */
+ std::vector<std::string> stopWords;
+ void readStopWords(const string languageCode);
+
+ /* Others */
+ unsigned int countWords(const string &text);
+
+ /* Boost factor */
+ unsigned int keywordsBoostFactor;
+ inline unsigned int getTitleBoostFactor(const unsigned int contentLength) {
+ return contentLength / 500 + 1;
+ }
+
+ /* Verbose */
+ pthread_mutex_t verboseMutex;
+ bool getVerboseFlag();
+ bool verboseFlag;
+
+ private:
+ pthread_mutex_t threadIdsMutex;
+
+ /* Index writting */
+ pthread_t articleIndexer;
+ pthread_mutex_t articleIndexerRunningMutex;
+ static void *indexArticles(void *ptr);
+ bool articleIndexerRunningFlag;
+ bool isArticleIndexerRunning();
+ void articleIndexerRunning(bool value);
+
+ /* To index queue */
+ std::queue<indexerToken> toIndexQueue;
+ pthread_mutex_t toIndexQueueMutex;
+ /* void pushToIndexQueue(indexerToken &token); is public */
+ bool popFromToIndexQueue(indexerToken &token);
+ bool isToIndexQueueEmpty();
+
+ /* Article Count & Progression */
+ unsigned int articleCount;
+ pthread_mutex_t articleCountMutex;
+ void setArticleCount(const unsigned int articleCount);
+ unsigned int getArticleCount();
+
+ /* Index path */
+ pthread_mutex_t indexPathMutex;
+ string indexPath;
+ void setIndexPath(const string path);
+ string getIndexPath();
+
+ /* ZIM id */
+ pthread_mutex_t zimIdMutex;
+ string zimId;
+ void setZimId(const string id);
+ string getZimId();
+ };
+
+#endif //OPENZIM_ZIMWRITERFS_INDEXER_H
diff --git a/zimwriterfs/pathTools.cpp b/zimwriterfs/pathTools.cpp
new file mode 100644
index 0000000..3570252
--- /dev/null
+++ b/zimwriterfs/pathTools.cpp
@@ -0,0 +1,78 @@
+/*
+ * Copyright 2011-2014 Emmanuel Engelhart <[email protected]>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ */
+
+#include "pathTools.h"
+
+#ifdef __APPLE__
+#include <mach-o/dyld.h>
+#include <limits.h>
+#elif _WIN32
+#include <windows.h>
+#include "Shlwapi.h"
+#endif
+
+#ifdef _WIN32
+#else
+#include <unistd.h>
+#endif
+
+#ifdef _WIN32
+#define SEPARATOR "\\"
+#else
+#define SEPARATOR "/"
+#endif
+
+#ifndef PATH_MAX
+#define PATH_MAX 1024
+#endif
+
+
+std::string appendToDirectory(const std::string &directoryPath, const
std::string &filename) {
+ std::string newPath = directoryPath + SEPARATOR + filename;
+ return newPath;
+}
+
+
+std::string getExecutablePath() {
+ char binRootPath[PATH_MAX];
+
+#ifdef _WIN32
+ GetModuleFileName( NULL, binRootPath, PATH_MAX);
+ return std::string(binRootPath);
+#elif __APPLE__
+ uint32_t max = (uint32_t)PATH_MAX;
+ _NSGetExecutablePath(binRootPath, &max);
+ return std::string(binRootPath);
+#else
+ ssize_t size = readlink("/proc/self/exe", binRootPath, PATH_MAX);
+ if (size != -1) {
+ return std::string(binRootPath, size);
+ }
+#endif
+
+ return "";
+}
+
+bool writeTextFile(const std::string &path, const std::string &content) {
+ std::ofstream file;
+ file.open(path.c_str());
+ file << content;
+ file.close();
+ return true;
+}
diff --git a/zimwriterfs/pathTools.h b/zimwriterfs/pathTools.h
new file mode 100644
index 0000000..b66486e
--- /dev/null
+++ b/zimwriterfs/pathTools.h
@@ -0,0 +1,46 @@
+/*
+ * Copyright 2011 Emmanuel Engelhart <[email protected]>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ */
+
+#ifndef OPENZIM_ZIMWRITERFS_PATHTOOLS_H
+#define OPENZIM_ZIMWRITERFS_PATHTOOLS_H
+
+#include <string>
+#include <vector>
+#include <sstream>
+#include <iostream>
+#include <fstream>
+#include <string.h>
+#include <stdio.h>
+#include <sys/types.h>
+#include <fcntl.h>
+#include <sys/stat.h>
+#include <ios>
+#include <limits.h>
+
+#ifdef _WIN32
+#include <direct.h>
+#endif
+
+std::string appendToDirectory(const std::string &directoryPath, const
std::string &filename);
+
+std::string getExecutablePath();
+
+bool writeTextFile(const std::string &path, const std::string &content);
+
+#endif //OPENZIM_ZIMWRITERFS_PATHTOOLS_H
diff --git a/zimwriterfs/resourceTools.cpp b/zimwriterfs/resourceTools.cpp
new file mode 100644
index 0000000..c00e5a0
--- /dev/null
+++ b/zimwriterfs/resourceTools.cpp
@@ -0,0 +1,380 @@
+
+#include <resourceTools.h>
+#include <string>
+#include <map>
+
+static const unsigned char stopwords_en[]={
+
0x61,0x0a,0x61,0x62,0x6c,0x65,0x0a,0x61,0x62,0x6f,0x75,0x74,0x0a,0x61,0x62,0x6f,
+
0x76,0x65,0x0a,0x61,0x62,0x73,0x74,0x0a,0x61,0x63,0x63,0x6f,0x72,0x64,0x61,0x6e,
+
0x63,0x65,0x0a,0x61,0x63,0x63,0x6f,0x72,0x64,0x69,0x6e,0x67,0x0a,0x61,0x63,0x63,
+
0x6f,0x72,0x64,0x69,0x6e,0x67,0x6c,0x79,0x0a,0x61,0x63,0x72,0x6f,0x73,0x73,0x0a,
+
0x61,0x63,0x74,0x0a,0x61,0x63,0x74,0x75,0x61,0x6c,0x6c,0x79,0x0a,0x61,0x64,0x64,
+
0x65,0x64,0x0a,0x61,0x64,0x6a,0x0a,0x61,0x64,0x6f,0x70,0x74,0x65,0x64,0x0a,0x61,
+
0x66,0x66,0x65,0x63,0x74,0x65,0x64,0x0a,0x61,0x66,0x66,0x65,0x63,0x74,0x69,0x6e,
+
0x67,0x0a,0x61,0x66,0x66,0x65,0x63,0x74,0x73,0x0a,0x61,0x66,0x74,0x65,0x72,0x0a,
+
0x61,0x66,0x74,0x65,0x72,0x77,0x61,0x72,0x64,0x73,0x0a,0x61,0x67,0x61,0x69,0x6e,
+
0x0a,0x61,0x67,0x61,0x69,0x6e,0x73,0x74,0x0a,0x61,0x68,0x0a,0x61,0x6c,0x6c,0x0a,
+
0x61,0x6c,0x6d,0x6f,0x73,0x74,0x0a,0x61,0x6c,0x6f,0x6e,0x65,0x0a,0x61,0x6c,0x6f,
+
0x6e,0x67,0x0a,0x61,0x6c,0x72,0x65,0x61,0x64,0x79,0x0a,0x61,0x6c,0x73,0x6f,0x0a,
+
0x61,0x6c,0x74,0x68,0x6f,0x75,0x67,0x68,0x0a,0x61,0x6c,0x77,0x61,0x79,0x73,0x0a,
+
0x61,0x6d,0x0a,0x61,0x6d,0x6f,0x6e,0x67,0x0a,0x61,0x6d,0x6f,0x6e,0x67,0x73,0x74,
+
0x0a,0x61,0x6e,0x0a,0x61,0x6e,0x64,0x0a,0x61,0x6e,0x6e,0x6f,0x75,0x6e,0x63,0x65,
+
0x0a,0x61,0x6e,0x6f,0x74,0x68,0x65,0x72,0x0a,0x61,0x6e,0x79,0x0a,0x61,0x6e,0x79,
+
0x62,0x6f,0x64,0x79,0x0a,0x61,0x6e,0x79,0x68,0x6f,0x77,0x0a,0x61,0x6e,0x79,0x6d,
+
0x6f,0x72,0x65,0x0a,0x61,0x6e,0x79,0x6f,0x6e,0x65,0x0a,0x61,0x6e,0x79,0x74,0x68,
+
0x69,0x6e,0x67,0x0a,0x61,0x6e,0x79,0x77,0x61,0x79,0x0a,0x61,0x6e,0x79,0x77,0x61,
+
0x79,0x73,0x0a,0x61,0x6e,0x79,0x77,0x68,0x65,0x72,0x65,0x0a,0x61,0x70,0x70,0x61,
+
0x72,0x65,0x6e,0x74,0x6c,0x79,0x0a,0x61,0x70,0x70,0x72,0x6f,0x78,0x69,0x6d,0x61,
+
0x74,0x65,0x6c,0x79,0x0a,0x61,0x72,0x65,0x0a,0x61,0x72,0x65,0x6e,0x0a,0x61,0x72,
+
0x65,0x6e,0x74,0x0a,0x61,0x72,0x69,0x73,0x65,0x0a,0x61,0x72,0x6f,0x75,0x6e,0x64,
+
0x0a,0x61,0x73,0x0a,0x61,0x73,0x69,0x64,0x65,0x0a,0x61,0x73,0x6b,0x0a,0x61,0x73,
+
0x6b,0x69,0x6e,0x67,0x0a,0x61,0x74,0x0a,0x61,0x75,0x74,0x68,0x0a,0x61,0x76,0x61,
+
0x69,0x6c,0x61,0x62,0x6c,0x65,0x0a,0x61,0x77,0x61,0x79,0x0a,0x61,0x77,0x66,0x75,
+
0x6c,0x6c,0x79,0x0a,0x62,0x0a,0x62,0x61,0x63,0x6b,0x0a,0x62,0x65,0x0a,0x62,0x65,
+
0x63,0x61,0x6d,0x65,0x0a,0x62,0x65,0x63,0x61,0x75,0x73,0x65,0x0a,0x62,0x65,0x63,
+
0x6f,0x6d,0x65,0x0a,0x62,0x65,0x63,0x6f,0x6d,0x65,0x73,0x0a,0x62,0x65,0x63,0x6f,
+
0x6d,0x69,0x6e,0x67,0x0a,0x62,0x65,0x65,0x6e,0x0a,0x62,0x65,0x66,0x6f,0x72,0x65,
+
0x0a,0x62,0x65,0x66,0x6f,0x72,0x65,0x68,0x61,0x6e,0x64,0x0a,0x62,0x65,0x67,0x69,
+
0x6e,0x0a,0x62,0x65,0x67,0x69,0x6e,0x6e,0x69,0x6e,0x67,0x0a,0x62,0x65,0x67,0x69,
+
0x6e,0x6e,0x69,0x6e,0x67,0x73,0x0a,0x62,0x65,0x67,0x69,0x6e,0x73,0x0a,0x62,0x65,
+
0x68,0x69,0x6e,0x64,0x0a,0x62,0x65,0x69,0x6e,0x67,0x0a,0x62,0x65,0x6c,0x69,0x65,
+
0x76,0x65,0x0a,0x62,0x65,0x6c,0x6f,0x77,0x0a,0x62,0x65,0x73,0x69,0x64,0x65,0x0a,
+
0x62,0x65,0x73,0x69,0x64,0x65,0x73,0x0a,0x62,0x65,0x74,0x77,0x65,0x65,0x6e,0x0a,
+
0x62,0x65,0x79,0x6f,0x6e,0x64,0x0a,0x62,0x69,0x6f,0x6c,0x0a,0x62,0x6f,0x74,0x68,
+
0x0a,0x62,0x72,0x69,0x65,0x66,0x0a,0x62,0x72,0x69,0x65,0x66,0x6c,0x79,0x0a,0x62,
+
0x75,0x74,0x0a,0x62,0x79,0x0a,0x63,0x0a,0x63,0x61,0x0a,0x63,0x61,0x6d,0x65,0x0a,
+
0x63,0x61,0x6e,0x0a,0x63,0x61,0x6e,0x6e,0x6f,0x74,0x0a,0x63,0x61,0x6e,0x27,0x74,
+
0x0a,0x63,0x61,0x75,0x73,0x65,0x0a,0x63,0x61,0x75,0x73,0x65,0x73,0x0a,0x63,0x65,
+
0x72,0x74,0x61,0x69,0x6e,0x0a,0x63,0x65,0x72,0x74,0x61,0x69,0x6e,0x6c,0x79,0x0a,
+
0x63,0x6f,0x0a,0x63,0x6f,0x6d,0x0a,0x63,0x6f,0x6d,0x65,0x0a,0x63,0x6f,0x6d,0x65,
+
0x73,0x0a,0x63,0x6f,0x6e,0x74,0x61,0x69,0x6e,0x0a,0x63,0x6f,0x6e,0x74,0x61,0x69,
+
0x6e,0x69,0x6e,0x67,0x0a,0x63,0x6f,0x6e,0x74,0x61,0x69,0x6e,0x73,0x0a,0x63,0x6f,
+
0x75,0x6c,0x64,0x0a,0x63,0x6f,0x75,0x6c,0x64,0x6e,0x74,0x0a,0x64,0x0a,0x64,0x61,
+
0x74,0x65,0x0a,0x64,0x69,0x64,0x0a,0x64,0x69,0x64,0x6e,0x27,0x74,0x0a,0x64,0x69,
+
0x66,0x66,0x65,0x72,0x65,0x6e,0x74,0x0a,0x64,0x6f,0x0a,0x64,0x6f,0x65,0x73,0x0a,
+
0x64,0x6f,0x65,0x73,0x6e,0x27,0x74,0x0a,0x64,0x6f,0x69,0x6e,0x67,0x0a,0x64,0x6f,
+
0x6e,0x65,0x0a,0x64,0x6f,0x6e,0x27,0x74,0x0a,0x64,0x6f,0x77,0x6e,0x0a,0x64,0x6f,
+
0x77,0x6e,0x77,0x61,0x72,0x64,0x73,0x0a,0x64,0x75,0x65,0x0a,0x64,0x75,0x72,0x69,
+
0x6e,0x67,0x0a,0x65,0x0a,0x65,0x61,0x63,0x68,0x0a,0x65,0x64,0x0a,0x65,0x64,0x75,
+
0x0a,0x65,0x66,0x66,0x65,0x63,0x74,0x0a,0x65,0x67,0x0a,0x65,0x69,0x67,0x68,0x74,
+
0x0a,0x65,0x69,0x67,0x68,0x74,0x79,0x0a,0x65,0x69,0x74,0x68,0x65,0x72,0x0a,0x65,
+
0x6c,0x73,0x65,0x0a,0x65,0x6c,0x73,0x65,0x77,0x68,0x65,0x72,0x65,0x0a,0x65,0x6e,
+
0x64,0x0a,0x65,0x6e,0x64,0x69,0x6e,0x67,0x0a,0x65,0x6e,0x6f,0x75,0x67,0x68,0x0a,
+
0x65,0x73,0x70,0x65,0x63,0x69,0x61,0x6c,0x6c,0x79,0x0a,0x65,0x74,0x0a,0x65,0x74,
+
0x2d,0x61,0x6c,0x0a,0x65,0x74,0x63,0x0a,0x65,0x76,0x65,0x6e,0x0a,0x65,0x76,0x65,
+
0x72,0x0a,0x65,0x76,0x65,0x72,0x79,0x0a,0x65,0x76,0x65,0x72,0x79,0x62,0x6f,0x64,
+
0x79,0x0a,0x65,0x76,0x65,0x72,0x79,0x6f,0x6e,0x65,0x0a,0x65,0x76,0x65,0x72,0x79,
+
0x74,0x68,0x69,0x6e,0x67,0x0a,0x65,0x76,0x65,0x72,0x79,0x77,0x68,0x65,0x72,0x65,
+
0x0a,0x65,0x78,0x0a,0x65,0x78,0x63,0x65,0x70,0x74,0x0a,0x66,0x0a,0x66,0x61,0x72,
+
0x0a,0x66,0x65,0x77,0x0a,0x66,0x66,0x0a,0x66,0x69,0x66,0x74,0x68,0x0a,0x66,0x69,
+
0x72,0x73,0x74,0x0a,0x66,0x69,0x76,0x65,0x0a,0x66,0x69,0x78,0x0a,0x66,0x6f,0x6c,
+
0x6c,0x6f,0x77,0x65,0x64,0x0a,0x66,0x6f,0x6c,0x6c,0x6f,0x77,0x69,0x6e,0x67,0x0a,
+
0x66,0x6f,0x6c,0x6c,0x6f,0x77,0x73,0x0a,0x66,0x6f,0x72,0x0a,0x66,0x6f,0x72,0x6d,
+
0x65,0x72,0x0a,0x66,0x6f,0x72,0x6d,0x65,0x72,0x6c,0x79,0x0a,0x66,0x6f,0x72,0x74,
+
0x68,0x0a,0x66,0x6f,0x75,0x6e,0x64,0x0a,0x66,0x6f,0x75,0x72,0x0a,0x66,0x72,0x6f,
+
0x6d,0x0a,0x66,0x75,0x72,0x74,0x68,0x65,0x72,0x0a,0x66,0x75,0x72,0x74,0x68,0x65,
+
0x72,0x6d,0x6f,0x72,0x65,0x0a,0x67,0x0a,0x67,0x61,0x76,0x65,0x0a,0x67,0x65,0x74,
+
0x0a,0x67,0x65,0x74,0x73,0x0a,0x67,0x65,0x74,0x74,0x69,0x6e,0x67,0x0a,0x67,0x69,
+
0x76,0x65,0x0a,0x67,0x69,0x76,0x65,0x6e,0x0a,0x67,0x69,0x76,0x65,0x73,0x0a,0x67,
+
0x69,0x76,0x69,0x6e,0x67,0x0a,0x67,0x6f,0x0a,0x67,0x6f,0x65,0x73,0x0a,0x67,0x6f,
+
0x6e,0x65,0x0a,0x67,0x6f,0x74,0x0a,0x67,0x6f,0x74,0x74,0x65,0x6e,0x0a,0x68,0x0a,
+
0x68,0x61,0x64,0x0a,0x68,0x61,0x70,0x70,0x65,0x6e,0x73,0x0a,0x68,0x61,0x72,0x64,
+
0x6c,0x79,0x0a,0x68,0x61,0x73,0x0a,0x68,0x61,0x73,0x6e,0x27,0x74,0x0a,0x68,0x61,
+
0x76,0x65,0x0a,0x68,0x61,0x76,0x65,0x6e,0x27,0x74,0x0a,0x68,0x61,0x76,0x69,0x6e,
+
0x67,0x0a,0x68,0x65,0x0a,0x68,0x65,0x64,0x0a,0x68,0x65,0x6e,0x63,0x65,0x0a,0x68,
+
0x65,0x72,0x0a,0x68,0x65,0x72,0x65,0x0a,0x68,0x65,0x72,0x65,0x61,0x66,0x74,0x65,
+
0x72,0x0a,0x68,0x65,0x72,0x65,0x62,0x79,0x0a,0x68,0x65,0x72,0x65,0x69,0x6e,0x0a,
+
0x68,0x65,0x72,0x65,0x73,0x0a,0x68,0x65,0x72,0x65,0x75,0x70,0x6f,0x6e,0x0a,0x68,
+
0x65,0x72,0x73,0x0a,0x68,0x65,0x72,0x73,0x65,0x6c,0x66,0x0a,0x68,0x65,0x73,0x0a,
+
0x68,0x69,0x0a,0x68,0x69,0x64,0x0a,0x68,0x69,0x6d,0x0a,0x68,0x69,0x6d,0x73,0x65,
+
0x6c,0x66,0x0a,0x68,0x69,0x73,0x0a,0x68,0x69,0x74,0x68,0x65,0x72,0x0a,0x68,0x6f,
+
0x6d,0x65,0x0a,0x68,0x6f,0x77,0x0a,0x68,0x6f,0x77,0x62,0x65,0x69,0x74,0x0a,0x68,
+
0x6f,0x77,0x65,0x76,0x65,0x72,0x0a,0x68,0x75,0x6e,0x64,0x72,0x65,0x64,0x0a,0x69,
+
0x0a,0x69,0x64,0x0a,0x69,0x65,0x0a,0x69,0x66,0x0a,0x69,0x27,0x6c,0x6c,0x0a,0x69,
+
0x6d,0x0a,0x69,0x6d,0x6d,0x65,0x64,0x69,0x61,0x74,0x65,0x0a,0x69,0x6d,0x6d,0x65,
+
0x64,0x69,0x61,0x74,0x65,0x6c,0x79,0x0a,0x69,0x6d,0x70,0x6f,0x72,0x74,0x61,0x6e,
+
0x63,0x65,0x0a,0x69,0x6d,0x70,0x6f,0x72,0x74,0x61,0x6e,0x74,0x0a,0x69,0x6e,0x0a,
+
0x69,0x6e,0x63,0x0a,0x69,0x6e,0x64,0x65,0x65,0x64,0x0a,0x69,0x6e,0x64,0x65,0x78,
+
0x0a,0x69,0x6e,0x66,0x6f,0x72,0x6d,0x61,0x74,0x69,0x6f,0x6e,0x0a,0x69,0x6e,0x73,
+
0x74,0x65,0x61,0x64,0x0a,0x69,0x6e,0x74,0x6f,0x0a,0x69,0x6e,0x76,0x65,0x6e,0x74,
+
0x69,0x6f,0x6e,0x0a,0x69,0x6e,0x77,0x61,0x72,0x64,0x0a,0x69,0x73,0x0a,0x69,0x73,
+
0x6e,0x27,0x74,0x0a,0x69,0x74,0x0a,0x69,0x74,0x64,0x0a,0x69,0x74,0x27,0x6c,0x6c,
+
0x0a,0x69,0x74,0x73,0x0a,0x69,0x74,0x73,0x65,0x6c,0x66,0x0a,0x69,0x27,0x76,0x65,
+
0x0a,0x6a,0x0a,0x6a,0x75,0x73,0x74,0x0a,0x6b,0x0a,0x6b,0x65,0x65,0x70,0x0a,0x6b,
+
0x65,0x65,0x70,0x73,0x0a,0x6b,0x65,0x70,0x74,0x0a,0x6b,0x65,0x79,0x73,0x0a,0x6b,
+
0x67,0x0a,0x6b,0x6d,0x0a,0x6b,0x6e,0x6f,0x77,0x0a,0x6b,0x6e,0x6f,0x77,0x6e,0x0a,
+
0x6b,0x6e,0x6f,0x77,0x73,0x0a,0x6c,0x0a,0x6c,0x61,0x72,0x67,0x65,0x6c,0x79,0x0a,
+
0x6c,0x61,0x73,0x74,0x0a,0x6c,0x61,0x74,0x65,0x6c,0x79,0x0a,0x6c,0x61,0x74,0x65,
+
0x72,0x0a,0x6c,0x61,0x74,0x74,0x65,0x72,0x0a,0x6c,0x61,0x74,0x74,0x65,0x72,0x6c,
+
0x79,0x0a,0x6c,0x65,0x61,0x73,0x74,0x0a,0x6c,0x65,0x73,0x73,0x0a,0x6c,0x65,0x73,
+
0x74,0x0a,0x6c,0x65,0x74,0x0a,0x6c,0x65,0x74,0x73,0x0a,0x6c,0x69,0x6b,0x65,0x0a,
+
0x6c,0x69,0x6b,0x65,0x64,0x0a,0x6c,0x69,0x6b,0x65,0x6c,0x79,0x0a,0x6c,0x69,0x6e,
+
0x65,0x0a,0x6c,0x69,0x74,0x74,0x6c,0x65,0x0a,0x27,0x6c,0x6c,0x0a,0x6c,0x6f,0x6f,
+
0x6b,0x0a,0x6c,0x6f,0x6f,0x6b,0x69,0x6e,0x67,0x0a,0x6c,0x6f,0x6f,0x6b,0x73,0x0a,
+
0x6c,0x74,0x64,0x0a,0x6d,0x0a,0x6d,0x61,0x64,0x65,0x0a,0x6d,0x61,0x69,0x6e,0x6c,
+
0x79,0x0a,0x6d,0x61,0x6b,0x65,0x0a,0x6d,0x61,0x6b,0x65,0x73,0x0a,0x6d,0x61,0x6e,
+
0x79,0x0a,0x6d,0x61,0x79,0x0a,0x6d,0x61,0x79,0x62,0x65,0x0a,0x6d,0x65,0x0a,0x6d,
+
0x65,0x61,0x6e,0x0a,0x6d,0x65,0x61,0x6e,0x73,0x0a,0x6d,0x65,0x61,0x6e,0x74,0x69,
+
0x6d,0x65,0x0a,0x6d,0x65,0x61,0x6e,0x77,0x68,0x69,0x6c,0x65,0x0a,0x6d,0x65,0x72,
+
0x65,0x6c,0x79,0x0a,0x6d,0x67,0x0a,0x6d,0x69,0x67,0x68,0x74,0x0a,0x6d,0x69,0x6c,
+
0x6c,0x69,0x6f,0x6e,0x0a,0x6d,0x69,0x73,0x73,0x0a,0x6d,0x6c,0x0a,0x6d,0x6f,0x72,
+
0x65,0x0a,0x6d,0x6f,0x72,0x65,0x6f,0x76,0x65,0x72,0x0a,0x6d,0x6f,0x73,0x74,0x0a,
+
0x6d,0x6f,0x73,0x74,0x6c,0x79,0x0a,0x6d,0x72,0x0a,0x6d,0x72,0x73,0x0a,0x6d,0x75,
+
0x63,0x68,0x0a,0x6d,0x75,0x67,0x0a,0x6d,0x75,0x73,0x74,0x0a,0x6d,0x79,0x0a,0x6d,
+
0x79,0x73,0x65,0x6c,0x66,0x0a,0x6e,0x0a,0x6e,0x61,0x0a,0x6e,0x61,0x6d,0x65,0x0a,
+
0x6e,0x61,0x6d,0x65,0x6c,0x79,0x0a,0x6e,0x61,0x79,0x0a,0x6e,0x64,0x0a,0x6e,0x65,
+
0x61,0x72,0x0a,0x6e,0x65,0x61,0x72,0x6c,0x79,0x0a,0x6e,0x65,0x63,0x65,0x73,0x73,
+
0x61,0x72,0x69,0x6c,0x79,0x0a,0x6e,0x65,0x63,0x65,0x73,0x73,0x61,0x72,0x79,0x0a,
+
0x6e,0x65,0x65,0x64,0x0a,0x6e,0x65,0x65,0x64,0x73,0x0a,0x6e,0x65,0x69,0x74,0x68,
+
0x65,0x72,0x0a,0x6e,0x65,0x76,0x65,0x72,0x0a,0x6e,0x65,0x76,0x65,0x72,0x74,0x68,
+
0x65,0x6c,0x65,0x73,0x73,0x0a,0x6e,0x65,0x77,0x0a,0x6e,0x65,0x78,0x74,0x0a,0x6e,
+
0x69,0x6e,0x65,0x0a,0x6e,0x69,0x6e,0x65,0x74,0x79,0x0a,0x6e,0x6f,0x0a,0x6e,0x6f,
+
0x62,0x6f,0x64,0x79,0x0a,0x6e,0x6f,0x6e,0x0a,0x6e,0x6f,0x6e,0x65,0x0a,0x6e,0x6f,
+
0x6e,0x65,0x74,0x68,0x65,0x6c,0x65,0x73,0x73,0x0a,0x6e,0x6f,0x6f,0x6e,0x65,0x0a,
+
0x6e,0x6f,0x72,0x0a,0x6e,0x6f,0x72,0x6d,0x61,0x6c,0x6c,0x79,0x0a,0x6e,0x6f,0x73,
+
0x0a,0x6e,0x6f,0x74,0x0a,0x6e,0x6f,0x74,0x65,0x64,0x0a,0x6e,0x6f,0x74,0x68,0x69,
+
0x6e,0x67,0x0a,0x6e,0x6f,0x77,0x0a,0x6e,0x6f,0x77,0x68,0x65,0x72,0x65,0x0a,0x6f,
+
0x0a,0x6f,0x62,0x74,0x61,0x69,0x6e,0x0a,0x6f,0x62,0x74,0x61,0x69,0x6e,0x65,0x64,
+
0x0a,0x6f,0x62,0x76,0x69,0x6f,0x75,0x73,0x6c,0x79,0x0a,0x6f,0x66,0x0a,0x6f,0x66,
+
0x66,0x0a,0x6f,0x66,0x74,0x65,0x6e,0x0a,0x6f,0x68,0x0a,0x6f,0x6b,0x0a,0x6f,0x6b,
+
0x61,0x79,0x0a,0x6f,0x6c,0x64,0x0a,0x6f,0x6d,0x69,0x74,0x74,0x65,0x64,0x0a,0x6f,
+
0x6e,0x0a,0x6f,0x6e,0x63,0x65,0x0a,0x6f,0x6e,0x65,0x0a,0x6f,0x6e,0x65,0x73,0x0a,
+
0x6f,0x6e,0x6c,0x79,0x0a,0x6f,0x6e,0x74,0x6f,0x0a,0x6f,0x72,0x0a,0x6f,0x72,0x64,
+
0x0a,0x6f,0x74,0x68,0x65,0x72,0x0a,0x6f,0x74,0x68,0x65,0x72,0x73,0x0a,0x6f,0x74,
+
0x68,0x65,0x72,0x77,0x69,0x73,0x65,0x0a,0x6f,0x75,0x67,0x68,0x74,0x0a,0x6f,0x75,
+
0x72,0x0a,0x6f,0x75,0x72,0x73,0x0a,0x6f,0x75,0x72,0x73,0x65,0x6c,0x76,0x65,0x73,
+
0x0a,0x6f,0x75,0x74,0x0a,0x6f,0x75,0x74,0x73,0x69,0x64,0x65,0x0a,0x6f,0x76,0x65,
+
0x72,0x0a,0x6f,0x76,0x65,0x72,0x61,0x6c,0x6c,0x0a,0x6f,0x77,0x69,0x6e,0x67,0x0a,
+
0x6f,0x77,0x6e,0x0a,0x70,0x0a,0x70,0x61,0x67,0x65,0x0a,0x70,0x61,0x67,0x65,0x73,
+
0x0a,0x70,0x61,0x72,0x74,0x0a,0x70,0x61,0x72,0x74,0x69,0x63,0x75,0x6c,0x61,0x72,
+
0x0a,0x70,0x61,0x72,0x74,0x69,0x63,0x75,0x6c,0x61,0x72,0x6c,0x79,0x0a,0x70,0x61,
+
0x73,0x74,0x0a,0x70,0x65,0x72,0x0a,0x70,0x65,0x72,0x68,0x61,0x70,0x73,0x0a,0x70,
+
0x6c,0x61,0x63,0x65,0x64,0x0a,0x70,0x6c,0x65,0x61,0x73,0x65,0x0a,0x70,0x6c,0x75,
+
0x73,0x0a,0x70,0x6f,0x6f,0x72,0x6c,0x79,0x0a,0x70,0x6f,0x73,0x73,0x69,0x62,0x6c,
+
0x65,0x0a,0x70,0x6f,0x73,0x73,0x69,0x62,0x6c,0x79,0x0a,0x70,0x6f,0x74,0x65,0x6e,
+
0x74,0x69,0x61,0x6c,0x6c,0x79,0x0a,0x70,0x70,0x0a,0x70,0x72,0x65,0x64,0x6f,0x6d,
+
0x69,0x6e,0x61,0x6e,0x74,0x6c,0x79,0x0a,0x70,0x72,0x65,0x73,0x65,0x6e,0x74,0x0a,
+
0x70,0x72,0x65,0x76,0x69,0x6f,0x75,0x73,0x6c,0x79,0x0a,0x70,0x72,0x69,0x6d,0x61,
+
0x72,0x69,0x6c,0x79,0x0a,0x70,0x72,0x6f,0x62,0x61,0x62,0x6c,0x79,0x0a,0x70,0x72,
+
0x6f,0x6d,0x70,0x74,0x6c,0x79,0x0a,0x70,0x72,0x6f,0x75,0x64,0x0a,0x70,0x72,0x6f,
+
0x76,0x69,0x64,0x65,0x73,0x0a,0x70,0x75,0x74,0x0a,0x71,0x0a,0x71,0x75,0x65,0x0a,
+
0x71,0x75,0x69,0x63,0x6b,0x6c,0x79,0x0a,0x71,0x75,0x69,0x74,0x65,0x0a,0x71,0x76,
+
0x0a,0x72,0x0a,0x72,0x61,0x6e,0x0a,0x72,0x61,0x74,0x68,0x65,0x72,0x0a,0x72,0x64,
+
0x0a,0x72,0x65,0x0a,0x72,0x65,0x61,0x64,0x69,0x6c,0x79,0x0a,0x72,0x65,0x61,0x6c,
+
0x6c,0x79,0x0a,0x72,0x65,0x63,0x65,0x6e,0x74,0x0a,0x72,0x65,0x63,0x65,0x6e,0x74,
+
0x6c,0x79,0x0a,0x72,0x65,0x66,0x0a,0x72,0x65,0x66,0x73,0x0a,0x72,0x65,0x67,0x61,
+
0x72,0x64,0x69,0x6e,0x67,0x0a,0x72,0x65,0x67,0x61,0x72,0x64,0x6c,0x65,0x73,0x73,
+
0x0a,0x72,0x65,0x67,0x61,0x72,0x64,0x73,0x0a,0x72,0x65,0x6c,0x61,0x74,0x65,0x64,
+
0x0a,0x72,0x65,0x6c,0x61,0x74,0x69,0x76,0x65,0x6c,0x79,0x0a,0x72,0x65,0x73,0x65,
+
0x61,0x72,0x63,0x68,0x0a,0x72,0x65,0x73,0x70,0x65,0x63,0x74,0x69,0x76,0x65,0x6c,
+
0x79,0x0a,0x72,0x65,0x73,0x75,0x6c,0x74,0x65,0x64,0x0a,0x72,0x65,0x73,0x75,0x6c,
+
0x74,0x69,0x6e,0x67,0x0a,0x72,0x65,0x73,0x75,0x6c,0x74,0x73,0x0a,0x72,0x69,0x67,
+
0x68,0x74,0x0a,0x72,0x75,0x6e,0x0a,0x73,0x0a,0x73,0x61,0x69,0x64,0x0a,0x73,0x61,
+
0x6d,0x65,0x0a,0x73,0x61,0x77,0x0a,0x73,0x61,0x79,0x0a,0x73,0x61,0x79,0x69,0x6e,
+
0x67,0x0a,0x73,0x61,0x79,0x73,0x0a,0x73,0x65,0x63,0x0a,0x73,0x65,0x63,0x74,0x69,
+
0x6f,0x6e,0x0a,0x73,0x65,0x65,0x0a,0x73,0x65,0x65,0x69,0x6e,0x67,0x0a,0x73,0x65,
+
0x65,0x6d,0x0a,0x73,0x65,0x65,0x6d,0x65,0x64,0x0a,0x73,0x65,0x65,0x6d,0x69,0x6e,
+
0x67,0x0a,0x73,0x65,0x65,0x6d,0x73,0x0a,0x73,0x65,0x65,0x6e,0x0a,0x73,0x65,0x6c,
+
0x66,0x0a,0x73,0x65,0x6c,0x76,0x65,0x73,0x0a,0x73,0x65,0x6e,0x74,0x0a,0x73,0x65,
+
0x76,0x65,0x6e,0x0a,0x73,0x65,0x76,0x65,0x72,0x61,0x6c,0x0a,0x73,0x68,0x61,0x6c,
+
0x6c,0x0a,0x73,0x68,0x65,0x0a,0x73,0x68,0x65,0x64,0x0a,0x73,0x68,0x65,0x27,0x6c,
+
0x6c,0x0a,0x73,0x68,0x65,0x73,0x0a,0x73,0x68,0x6f,0x75,0x6c,0x64,0x0a,0x73,0x68,
+
0x6f,0x75,0x6c,0x64,0x6e,0x27,0x74,0x0a,0x73,0x68,0x6f,0x77,0x0a,0x73,0x68,0x6f,
+
0x77,0x65,0x64,0x0a,0x73,0x68,0x6f,0x77,0x6e,0x0a,0x73,0x68,0x6f,0x77,0x6e,0x73,
+
0x0a,0x73,0x68,0x6f,0x77,0x73,0x0a,0x73,0x69,0x67,0x6e,0x69,0x66,0x69,0x63,0x61,
+
0x6e,0x74,0x0a,0x73,0x69,0x67,0x6e,0x69,0x66,0x69,0x63,0x61,0x6e,0x74,0x6c,0x79,
+
0x0a,0x73,0x69,0x6d,0x69,0x6c,0x61,0x72,0x0a,0x73,0x69,0x6d,0x69,0x6c,0x61,0x72,
+
0x6c,0x79,0x0a,0x73,0x69,0x6e,0x63,0x65,0x0a,0x73,0x69,0x78,0x0a,0x73,0x6c,0x69,
+
0x67,0x68,0x74,0x6c,0x79,0x0a,0x73,0x6f,0x0a,0x73,0x6f,0x6d,0x65,0x0a,0x73,0x6f,
+
0x6d,0x65,0x62,0x6f,0x64,0x79,0x0a,0x73,0x6f,0x6d,0x65,0x68,0x6f,0x77,0x0a,0x73,
+
0x6f,0x6d,0x65,0x6f,0x6e,0x65,0x0a,0x73,0x6f,0x6d,0x65,0x74,0x68,0x61,0x6e,0x0a,
+
0x73,0x6f,0x6d,0x65,0x74,0x68,0x69,0x6e,0x67,0x0a,0x73,0x6f,0x6d,0x65,0x74,0x69,
+
0x6d,0x65,0x0a,0x73,0x6f,0x6d,0x65,0x74,0x69,0x6d,0x65,0x73,0x0a,0x73,0x6f,0x6d,
+
0x65,0x77,0x68,0x61,0x74,0x0a,0x73,0x6f,0x6d,0x65,0x77,0x68,0x65,0x72,0x65,0x0a,
+
0x73,0x6f,0x6f,0x6e,0x0a,0x73,0x6f,0x72,0x72,0x79,0x0a,0x73,0x70,0x65,0x63,0x69,
+
0x66,0x69,0x63,0x61,0x6c,0x6c,0x79,0x0a,0x73,0x70,0x65,0x63,0x69,0x66,0x69,0x65,
+
0x64,0x0a,0x73,0x70,0x65,0x63,0x69,0x66,0x79,0x0a,0x73,0x70,0x65,0x63,0x69,0x66,
+
0x79,0x69,0x6e,0x67,0x0a,0x73,0x74,0x61,0x74,0x65,0x0a,0x73,0x74,0x61,0x74,0x65,
+
0x73,0x0a,0x73,0x74,0x69,0x6c,0x6c,0x0a,0x73,0x74,0x6f,0x70,0x0a,0x73,0x74,0x72,
+
0x6f,0x6e,0x67,0x6c,0x79,0x0a,0x73,0x75,0x62,0x0a,0x73,0x75,0x62,0x73,0x74,0x61,
+
0x6e,0x74,0x69,0x61,0x6c,0x6c,0x79,0x0a,0x73,0x75,0x63,0x63,0x65,0x73,0x73,0x66,
+
0x75,0x6c,0x6c,0x79,0x0a,0x73,0x75,0x63,0x68,0x0a,0x73,0x75,0x66,0x66,0x69,0x63,
+
0x69,0x65,0x6e,0x74,0x6c,0x79,0x0a,0x73,0x75,0x67,0x67,0x65,0x73,0x74,0x0a,0x73,
+
0x75,0x70,0x0a,0x73,0x75,0x72,0x65,0x0a,0x09,0x74,0x0a,0x74,0x61,0x6b,0x65,0x0a,
+
0x74,0x61,0x6b,0x65,0x6e,0x0a,0x74,0x61,0x6b,0x69,0x6e,0x67,0x0a,0x74,0x65,0x6c,
+
0x6c,0x0a,0x74,0x65,0x6e,0x64,0x73,0x0a,0x74,0x68,0x0a,0x74,0x68,0x61,0x6e,0x0a,
+
0x74,0x68,0x61,0x6e,0x6b,0x0a,0x74,0x68,0x61,0x6e,0x6b,0x73,0x0a,0x74,0x68,0x61,
+
0x6e,0x78,0x0a,0x74,0x68,0x61,0x74,0x0a,0x74,0x68,0x61,0x74,0x27,0x6c,0x6c,0x0a,
+
0x74,0x68,0x61,0x74,0x73,0x0a,0x74,0x68,0x61,0x74,0x27,0x76,0x65,0x0a,0x74,0x68,
+
0x65,0x0a,0x74,0x68,0x65,0x69,0x72,0x0a,0x74,0x68,0x65,0x69,0x72,0x73,0x0a,0x74,
+
0x68,0x65,0x6d,0x0a,0x74,0x68,0x65,0x6d,0x73,0x65,0x6c,0x76,0x65,0x73,0x0a,0x74,
+
0x68,0x65,0x6e,0x0a,0x74,0x68,0x65,0x6e,0x63,0x65,0x0a,0x74,0x68,0x65,0x72,0x65,
+
0x0a,0x74,0x68,0x65,0x72,0x65,0x61,0x66,0x74,0x65,0x72,0x0a,0x74,0x68,0x65,0x72,
+
0x65,0x62,0x79,0x0a,0x74,0x68,0x65,0x72,0x65,0x64,0x0a,0x74,0x68,0x65,0x72,0x65,
+
0x66,0x6f,0x72,0x65,0x0a,0x74,0x68,0x65,0x72,0x65,0x69,0x6e,0x0a,0x74,0x68,0x65,
+
0x72,0x65,0x27,0x6c,0x6c,0x0a,0x74,0x68,0x65,0x72,0x65,0x6f,0x66,0x0a,0x74,0x68,
+
0x65,0x72,0x65,0x72,0x65,0x0a,0x74,0x68,0x65,0x72,0x65,0x73,0x0a,0x74,0x68,0x65,
+
0x72,0x65,0x74,0x6f,0x0a,0x74,0x68,0x65,0x72,0x65,0x75,0x70,0x6f,0x6e,0x0a,0x74,
+
0x68,0x65,0x72,0x65,0x27,0x76,0x65,0x0a,0x74,0x68,0x65,0x73,0x65,0x0a,0x74,0x68,
+
0x65,0x79,0x0a,0x74,0x68,0x65,0x79,0x64,0x0a,0x74,0x68,0x65,0x79,0x27,0x6c,0x6c,
+
0x0a,0x74,0x68,0x65,0x79,0x72,0x65,0x0a,0x74,0x68,0x65,0x79,0x27,0x76,0x65,0x0a,
+
0x74,0x68,0x69,0x6e,0x6b,0x0a,0x74,0x68,0x69,0x73,0x0a,0x74,0x68,0x6f,0x73,0x65,
+
0x0a,0x74,0x68,0x6f,0x75,0x0a,0x74,0x68,0x6f,0x75,0x67,0x68,0x0a,0x74,0x68,0x6f,
+
0x75,0x67,0x68,0x68,0x0a,0x74,0x68,0x6f,0x75,0x73,0x61,0x6e,0x64,0x0a,0x74,0x68,
+
0x72,0x6f,0x75,0x67,0x0a,0x74,0x68,0x72,0x6f,0x75,0x67,0x68,0x0a,0x74,0x68,0x72,
+
0x6f,0x75,0x67,0x68,0x6f,0x75,0x74,0x0a,0x74,0x68,0x72,0x75,0x0a,0x74,0x68,0x75,
+
0x73,0x0a,0x74,0x69,0x6c,0x0a,0x74,0x69,0x70,0x0a,0x74,0x6f,0x0a,0x74,0x6f,0x67,
+
0x65,0x74,0x68,0x65,0x72,0x0a,0x74,0x6f,0x6f,0x0a,0x74,0x6f,0x6f,0x6b,0x0a,0x74,
+
0x6f,0x77,0x61,0x72,0x64,0x0a,0x74,0x6f,0x77,0x61,0x72,0x64,0x73,0x0a,0x74,0x72,
+
0x69,0x65,0x64,0x0a,0x74,0x72,0x69,0x65,0x73,0x0a,0x74,0x72,0x75,0x6c,0x79,0x0a,
+
0x74,0x72,0x79,0x0a,0x74,0x72,0x79,0x69,0x6e,0x67,0x0a,0x74,0x73,0x0a,0x74,0x77,
+
0x69,0x63,0x65,0x0a,0x74,0x77,0x6f,0x0a,0x75,0x0a,0x75,0x6e,0x0a,0x75,0x6e,0x64,
+
0x65,0x72,0x0a,0x75,0x6e,0x66,0x6f,0x72,0x74,0x75,0x6e,0x61,0x74,0x65,0x6c,0x79,
+
0x0a,0x75,0x6e,0x6c,0x65,0x73,0x73,0x0a,0x75,0x6e,0x6c,0x69,0x6b,0x65,0x0a,0x75,
+
0x6e,0x6c,0x69,0x6b,0x65,0x6c,0x79,0x0a,0x75,0x6e,0x74,0x69,0x6c,0x0a,0x75,0x6e,
+
0x74,0x6f,0x0a,0x75,0x70,0x0a,0x75,0x70,0x6f,0x6e,0x0a,0x75,0x70,0x73,0x0a,0x75,
+
0x73,0x0a,0x75,0x73,0x65,0x0a,0x75,0x73,0x65,0x64,0x0a,0x75,0x73,0x65,0x66,0x75,
+
0x6c,0x0a,0x75,0x73,0x65,0x66,0x75,0x6c,0x6c,0x79,0x0a,0x75,0x73,0x65,0x66,0x75,
+
0x6c,0x6e,0x65,0x73,0x73,0x0a,0x75,0x73,0x65,0x73,0x0a,0x75,0x73,0x69,0x6e,0x67,
+
0x0a,0x75,0x73,0x75,0x61,0x6c,0x6c,0x79,0x0a,0x76,0x0a,0x76,0x61,0x6c,0x75,0x65,
+
0x0a,0x76,0x61,0x72,0x69,0x6f,0x75,0x73,0x0a,0x27,0x76,0x65,0x0a,0x76,0x65,0x72,
+
0x79,0x0a,0x76,0x69,0x61,0x0a,0x76,0x69,0x7a,0x0a,0x76,0x6f,0x6c,0x0a,0x76,0x6f,
+
0x6c,0x73,0x0a,0x76,0x73,0x0a,0x77,0x0a,0x77,0x61,0x6e,0x74,0x0a,0x77,0x61,0x6e,
+
0x74,0x73,0x0a,0x77,0x61,0x73,0x0a,0x77,0x61,0x73,0x6e,0x27,0x74,0x0a,0x77,0x61,
+
0x79,0x0a,0x77,0x65,0x0a,0x77,0x65,0x64,0x0a,0x77,0x65,0x6c,0x63,0x6f,0x6d,0x65,
+
0x0a,0x77,0x65,0x27,0x6c,0x6c,0x0a,0x77,0x65,0x6e,0x74,0x0a,0x77,0x65,0x72,0x65,
+
0x0a,0x77,0x65,0x72,0x65,0x6e,0x27,0x74,0x0a,0x77,0x65,0x27,0x76,0x65,0x0a,0x77,
+
0x68,0x61,0x74,0x0a,0x77,0x68,0x61,0x74,0x65,0x76,0x65,0x72,0x0a,0x77,0x68,0x61,
+
0x74,0x27,0x6c,0x6c,0x0a,0x77,0x68,0x61,0x74,0x73,0x0a,0x77,0x68,0x65,0x6e,0x0a,
+
0x77,0x68,0x65,0x6e,0x63,0x65,0x0a,0x77,0x68,0x65,0x6e,0x65,0x76,0x65,0x72,0x0a,
+
0x77,0x68,0x65,0x72,0x65,0x0a,0x77,0x68,0x65,0x72,0x65,0x61,0x66,0x74,0x65,0x72,
+
0x0a,0x77,0x68,0x65,0x72,0x65,0x61,0x73,0x0a,0x77,0x68,0x65,0x72,0x65,0x62,0x79,
+
0x0a,0x77,0x68,0x65,0x72,0x65,0x69,0x6e,0x0a,0x77,0x68,0x65,0x72,0x65,0x73,0x0a,
+
0x77,0x68,0x65,0x72,0x65,0x75,0x70,0x6f,0x6e,0x0a,0x77,0x68,0x65,0x72,0x65,0x76,
+
0x65,0x72,0x0a,0x77,0x68,0x65,0x74,0x68,0x65,0x72,0x0a,0x77,0x68,0x69,0x63,0x68,
+
0x0a,0x77,0x68,0x69,0x6c,0x65,0x0a,0x77,0x68,0x69,0x6d,0x0a,0x77,0x68,0x69,0x74,
+
0x68,0x65,0x72,0x0a,0x77,0x68,0x6f,0x0a,0x77,0x68,0x6f,0x64,0x0a,0x77,0x68,0x6f,
+
0x65,0x76,0x65,0x72,0x0a,0x77,0x68,0x6f,0x6c,0x65,0x0a,0x77,0x68,0x6f,0x27,0x6c,
+
0x6c,0x0a,0x77,0x68,0x6f,0x6d,0x0a,0x77,0x68,0x6f,0x6d,0x65,0x76,0x65,0x72,0x0a,
+
0x77,0x68,0x6f,0x73,0x0a,0x77,0x68,0x6f,0x73,0x65,0x0a,0x77,0x68,0x79,0x0a,0x77,
+
0x69,0x64,0x65,0x6c,0x79,0x0a,0x77,0x69,0x6c,0x6c,0x69,0x6e,0x67,0x0a,0x77,0x69,
+
0x73,0x68,0x0a,0x77,0x69,0x74,0x68,0x0a,0x77,0x69,0x74,0x68,0x69,0x6e,0x0a,0x77,
+
0x69,0x74,0x68,0x6f,0x75,0x74,0x0a,0x77,0x6f,0x6e,0x27,0x74,0x0a,0x77,0x6f,0x72,
+
0x64,0x73,0x0a,0x77,0x6f,0x72,0x6c,0x64,0x0a,0x77,0x6f,0x75,0x6c,0x64,0x0a,0x77,
+
0x6f,0x75,0x6c,0x64,0x6e,0x27,0x74,0x0a,0x77,0x77,0x77,0x0a,0x78,0x0a,0x79,0x0a,
+
0x79,0x65,0x73,0x0a,0x79,0x65,0x74,0x0a,0x79,0x6f,0x75,0x0a,0x79,0x6f,0x75,0x64,
+
0x0a,0x79,0x6f,0x75,0x27,0x6c,0x6c,0x0a,0x79,0x6f,0x75,0x72,0x0a,0x79,0x6f,0x75,
+
0x72,0x65,0x0a,0x79,0x6f,0x75,0x72,0x73,0x0a,0x79,0x6f,0x75,0x72,0x73,0x65,0x6c,
+
0x66,0x0a,0x79,0x6f,0x75,0x72,0x73,0x65,0x6c,0x76,0x65,0x73,0x0a,0x79,0x6f,0x75,
+ 0x27,0x76,0x65,0x0a,0x7a,0x0a,0x7a,0x65,0x72,0x6f,0x0a
+ };
+
+static const unsigned char stopwords_fra[]={
+
0x61,0x6c,0x6f,0x72,0x73,0x0a,0x61,0x75,0x0a,0x61,0x75,0x63,0x75,0x6e,0x73,0x0a,
+
0x61,0x75,0x73,0x73,0x69,0x0a,0x61,0x75,0x74,0x72,0x65,0x0a,0x61,0x76,0x61,0x6e,
+
0x74,0x0a,0x61,0x76,0x65,0x63,0x0a,0x61,0x76,0x6f,0x69,0x72,0x0a,0x62,0x6f,0x6e,
+
0x0a,0x63,0x61,0x72,0x0a,0x63,0x65,0x0a,0x63,0x65,0x6c,0x61,0x0a,0x63,0x65,0x73,
+
0x0a,0x63,0x65,0x75,0x78,0x0a,0x63,0x68,0x61,0x71,0x75,0x65,0x0a,0x63,0x69,0x0a,
+
0x63,0x6f,0x6d,0x6d,0x65,0x0a,0x63,0x6f,0x6d,0x6d,0x65,0x6e,0x74,0x0a,0x64,0x61,
+
0x6e,0x73,0x0a,0x64,0x65,0x73,0x0a,0x64,0x75,0x0a,0x64,0x65,0x64,0x61,0x6e,0x73,
+
0x0a,0x64,0x65,0x68,0x6f,0x72,0x73,0x0a,0x64,0x65,0x70,0x75,0x69,0x73,0x0a,0x64,
+
0x65,0x75,0x78,0x0a,0x64,0x65,0x76,0x72,0x61,0x69,0x74,0x0a,0x64,0x6f,0x69,0x74,
+
0x0a,0x64,0x6f,0x6e,0x63,0x0a,0x64,0x6f,0x73,0x0a,0x64,0x72,0x6f,0x69,0x74,0x65,
+
0x0a,0x64,0xc3,0xa9,0x62,0x75,0x74,0x0a,0x65,0x6c,0x6c,0x65,0x0a,0x65,0x6c,0x6c,
+
0x65,0x73,0x0a,0x65,0x6e,0x0a,0x65,0x6e,0x63,0x6f,0x72,0x65,0x0a,0x65,0x73,0x73,
+
0x61,0x69,0x0a,0x65,0x73,0x74,0x0a,0x65,0x74,0x0a,0x65,0x75,0x0a,0x66,0x61,0x69,
+
0x74,0x0a,0x66,0x61,0x69,0x74,0x65,0x73,0x0a,0x66,0x6f,0x69,0x73,0x0a,0x66,0x6f,
+
0x6e,0x74,0x0a,0x66,0x6f,0x72,0x63,0x65,0x0a,0x68,0x61,0x75,0x74,0x0a,0x68,0x6f,
+
0x72,0x73,0x0a,0x69,0x63,0x69,0x0a,0x69,0x6c,0x0a,0x69,0x6c,0x73,0x0a,0x6a,0x65,
+
0x0a,0x6c,0x61,0x0a,0x6c,0x65,0x0a,0x6c,0x65,0x73,0x0a,0x6c,0x65,0x75,0x72,0x0a,
+
0x6c,0xc3,0xa0,0x0a,0x6d,0x61,0x0a,0x6d,0x61,0x69,0x6e,0x74,0x65,0x6e,0x61,0x6e,
+
0x74,0x0a,0x6d,0x61,0x69,0x73,0x0a,0x6d,0x65,0x73,0x0a,0x6d,0x69,0x6e,0x65,0x0a,
+
0x6d,0x6f,0x69,0x6e,0x73,0x0a,0x6d,0x6f,0x6e,0x0a,0x6d,0x6f,0x74,0x0a,0x6d,0xc3,
+
0xaa,0x6d,0x65,0x0a,0x6e,0x69,0x0a,0x6e,0x6f,0x6d,0x6d,0xc3,0xa9,0x73,0x0a,0x6e,
+
0x6f,0x74,0x72,0x65,0x0a,0x6e,0x6f,0x75,0x73,0x0a,0x6e,0x6f,0x75,0x76,0x65,0x61,
+
0x75,0x78,0x0a,0x6f,0x75,0x0a,0x6f,0xc3,0xb9,0x0a,0x70,0x61,0x72,0x0a,0x70,0x61,
+
0x72,0x63,0x65,0x0a,0x70,0x61,0x72,0x6f,0x6c,0x65,0x0a,0x70,0x61,0x73,0x0a,0x70,
+
0x65,0x72,0x73,0x6f,0x6e,0x6e,0x65,0x73,0x0a,0x70,0x65,0x75,0x74,0x0a,0x70,0x65,
+
0x75,0x0a,0x70,0x69,0xc3,0xa8,0x63,0x65,0x0a,0x70,0x6c,0x75,0x70,0x61,0x72,0x74,
+
0x0a,0x70,0x6f,0x75,0x72,0x0a,0x70,0x6f,0x75,0x72,0x71,0x75,0x6f,0x69,0x0a,0x71,
+
0x75,0x61,0x6e,0x64,0x0a,0x71,0x75,0x65,0x0a,0x71,0x75,0x65,0x6c,0x0a,0x71,0x75,
+
0x65,0x6c,0x6c,0x65,0x0a,0x71,0x75,0x65,0x6c,0x6c,0x65,0x73,0x0a,0x71,0x75,0x65,
+
0x6c,0x73,0x0a,0x71,0x75,0x69,0x0a,0x73,0x61,0x0a,0x73,0x61,0x6e,0x73,0x0a,0x73,
+
0x65,0x73,0x0a,0x73,0x65,0x75,0x6c,0x65,0x6d,0x65,0x6e,0x74,0x0a,0x73,0x69,0x0a,
+
0x73,0x69,0x65,0x6e,0x0a,0x73,0x6f,0x6e,0x0a,0x73,0x6f,0x6e,0x74,0x0a,0x73,0x6f,
+
0x75,0x73,0x0a,0x73,0x6f,0x79,0x65,0x7a,0x0a,0x73,0x75,0x72,0x0a,0x74,0x61,0x0a,
+
0x74,0x61,0x6e,0x64,0x69,0x73,0x0a,0x74,0x65,0x6c,0x6c,0x65,0x6d,0x65,0x6e,0x74,
+
0x0a,0x74,0x65,0x6c,0x73,0x0a,0x74,0x65,0x73,0x0a,0x74,0x6f,0x6e,0x0a,0x74,0x6f,
+
0x75,0x73,0x0a,0x74,0x6f,0x75,0x74,0x0a,0x74,0x72,0x6f,0x70,0x0a,0x74,0x72,0xc3,
+
0xa8,0x73,0x0a,0x74,0x75,0x0a,0x76,0x61,0x6c,0x65,0x75,0x72,0x0a,0x76,0x6f,0x69,
+
0x65,0x0a,0x76,0x6f,0x69,0x65,0x6e,0x74,0x0a,0x76,0x6f,0x6e,0x74,0x0a,0x76,0x6f,
+
0x74,0x72,0x65,0x0a,0x76,0x6f,0x75,0x73,0x0a,0x76,0x75,0x0a,0xc3,0xa7,0x61,0x0a,
+
0xc3,0xa9,0x74,0x61,0x69,0x65,0x6e,0x74,0x0a,0xc3,0xa9,0x74,0x61,0x74,0x0a,0xc3,
+
0xa9,0x74,0x69,0x6f,0x6e,0x73,0x0a,0xc3,0xa9,0x74,0xc3,0xa9,0x0a,0xc3,0xaa,0x74,
+ 0x72,0x65,0x0a
+ };
+
+static const unsigned char stopwords_he[]={
+
0xd7,0xa9,0xd7,0x9c,0x0d,0x0a,0xd7,0x90,0xd7,0xaa,0x0d,0x0a,0xd7,0xa2,0xd7,0x9c,
+
0x0d,0x0a,0xd7,0x9c,0xd7,0x90,0x0d,0x0a,0xd7,0x9b,0xd7,0x99,0x0d,0x0a,0xd7,0xa2,
+
0xd7,0x9d,0x0d,0x0a,0xd7,0x94,0xd7,0x95,0xd7,0x90,0x0d,0x0a,0xd7,0x92,0xd7,0x9d,
+
0x0d,0x0a,0xd7,0x91,0x0d,0x0a,0xd7,0x96,0xd7,0x94,0x0d,0x0a,0xd7,0x94,0xd7,0x99,
+
0xd7,0x90,0x0d,0x0a,0xd7,0x9b,0xd7,0x9c,0x0d,0x0a,0xd7,0x99,0xd7,0x95,0xd7,0xaa,
+
0xd7,0xa8,0x0d,0x0a,0xd7,0x90,0xd7,0x95,0x0d,0x0a,0xd7,0x90,0xd7,0x91,0xd7,0x9c,
+
0x0d,0x0a,0xd7,0x91,0xd7,0x99,0xd7,0x9f,0x0d,0x0a,0xd7,0x94,0xd7,0x99,0xd7,0x94,
+
0x0d,0x0a,0xd7,0x90,0xd7,0x9d,0x0d,0x0a,0xd7,0x9e,0xd7,0x99,0xd7,0x9c,0xd7,0x99,
+
0xd7,0x95,0xd7,0x9f,0x0d,0x0a,0xd7,0x99,0xd7,0xa9,0x0d,0x0a,0xd7,0x9b,0xd7,0x9a,
+
0x0d,0x0a,0xd7,0x90,0xd7,0xa0,0xd7,0x99,0x0d,0x0a,0xd7,0x94,0xd7,0x9d,0x0d,0x0a,
+
0xd7,0x93,0xd7,0x95,0xd7,0x9c,0xd7,0xa8,0x0d,0x0a,0xd7,0x90,0xd7,0x9e,0xd7,0xa8,
+
0x0d,0x0a,0xd7,0xa2,0xd7,0x93,0x0d,0x0a,0xd7,0x9c,0xd7,0x90,0xd7,0x97,0xd7,0xa8,
+
0x0d,0x0a,0xd7,0x99,0xd7,0xa9,0xd7,0xa8,0xd7,0x90,0xd7,0x9c,0x0d,0x0a,0xd7,0xa8,
+
0xd7,0xa7,0x0d,0x0a,0xd7,0xa9,0xd7,0xa7,0xd7,0x9c,0x0d,0x0a,0xd7,0x9b,0xd7,0x93,
+
0xd7,0x99,0x0d,0x0a,0xd7,0x9e,0xd7,0x94,0x0d,0x0a,0xd7,0x9c,0xd7,0xa4,0xd7,0xa0,
+
0xd7,0x99,0x0d,0x0a,0xd7,0x90,0xd7,0x97,0xd7,0x93,0x0d,0x0a,0xd7,0x94,0xd7,0x97,
+
0xd7,0x91,0xd7,0xa8,0xd7,0x94,0x0d,0x0a,0xd7,0x9b,0xd7,0x9e,0xd7,0x95,0x0d,0x0a,
+
0xd7,0x96,0xd7,0x90,0xd7,0xaa,0x0d,0x0a,0xd7,0x94,0xd7,0x99,0xd7,0x95,0xd7,0x9d,
+
0x0d,0x0a,0xd7,0x90,0xd7,0x9a,0x0d,0x0a,0xd7,0x9c,0x0d,0x0a,0xd7,0x94,0x0d,0x0a,
+
0xd7,0x9b,0x0d,0x0a,0xd7,0x90,0xd7,0x99,0xd7,0x9f,0x0d,0x0a,0xd7,0x90,0xd7,0xaa,
+
0xd7,0x9e,0xd7,0x95,0xd7,0x9c,0x0d,0x0a,0xd7,0xa9,0xd7,0x9c,0xd7,0x90,0x0d,0x0a,
+
0xd7,0x9b,0xd7,0x91,0xd7,0xa8,0x0d,0x0a,0xd7,0xa2,0xd7,0x95,0xd7,0x93,0x0d,0x0a,
+
0xd7,0x9c,0xd7,0x95,0x0d,0x0a,0xd7,0x96,0xd7,0x95,0x0d,0x0a,0xd7,0x90,0xd7,0x9c,
+
0x0d,0x0a,0xd7,0x91,0xd7,0x9f,0x0d,0x0a,0xd7,0x90,0xd7,0x95,0xd7,0xaa,0xd7,0x95,
+
0x0d,0x0a,0xd7,0xa9,0xd7,0xa0,0xd7,0x99,0x0d,0x0a,0xd7,0x91,0xd7,0x99,0xd7,0xaa,
+
0x0d,0x0a,0xd7,0x99,0xd7,0x93,0xd7,0x99,0x0d,0x0a,0xd7,0x9b,0xd7,0x9e,0xd7,0x94,
+
0x0d,0x0a,0xd7,0x91,0xd7,0x99,0xd7,0x95,0xd7,0xaa,0xd7,0xa8,0x0d,0x0a,0xd7,0x95,
+
0xd7,0x9c,0xd7,0x90,0x0d,0x0a,0xd7,0x94,0xd7,0x9e,0xd7,0x9e,0xd7,0xa9,0xd7,0x9c,
+
0xd7,0x94,0x0d,0x0a,0xd7,0x90,0xd7,0x97,0xd7,0xa8,0xd7,0x99,0x0d,0x0a,0xd7,0x97,
+
0xd7,0x91,0xd7,0xa8,0xd7,0xaa,0x0d,0x0a,0xd7,0x94,0xd7,0x99,0xd7,0xaa,0xd7,0x94,
+
0x0d,0x0a,0xd7,0xa9,0xd7,0x9c,0xd7,0x95,0x0d,0x0a,0xd7,0x94,0xd7,0x99,0xd7,0x95,
+
0x0d,0x0a,0xd7,0xa0,0xd7,0x92,0xd7,0x93,0x0d,0x0a,0xd7,0x91,0xd7,0x9b,0xd7,0x9c,
+
0x0d,0x0a,0xd7,0x90,0xd7,0x91,0xd7,0x99,0xd7,0x91,0x0d,0x0a,0xd7,0xa8,0xd7,0x90,
+
0xd7,0xa9,0x0d,0x0a,0xd7,0x91,0xd7,0x99,0xd7,0xa9,0xd7,0xa8,0xd7,0x90,0xd7,0x9c,
+
0x0d,0x0a,0xd7,0x9c,0xd7,0x99,0x0d,0x0a,0xd7,0xa9,0xd7,0xa0,0xd7,0x99,0xd7,0x9d,
+
0x0d,0x0a,0xd7,0xa4,0xd7,0x99,0x0d,0x0a,0xd7,0x91,0xd7,0x95,0x0d,0x0a,0xd7,0x9e,
+
0x0d,0x0a,0xd7,0x9e,0xd7,0x90,0xd7,0x95,0xd7,0x93,0x0d,0x0a,0xd7,0x9c,0xd7,0x94,
+
0xd7,0x99,0xd7,0x95,0xd7,0xaa,0x0d,0x0a,0xd7,0xa9,0xd7,0x94,0xd7,0x95,0xd7,0x90,
+
0x0d,0x0a,0xd7,0x9e,0xd7,0x99,0x0d,0x0a,0xd7,0x90,0xd7,0x9c,0xd7,0xa3,0x0d,0x0a,
+
0xd7,0x90,0xd7,0x9c,0xd7,0x90,0x0d,0x0a,0xd7,0x90,0xd7,0xa3,0x0d,0x0a,0xd7,0x90,
+
0xd7,0x97,0xd7,0xa8,0x0d,0x0a,0xd7,0x94,0xd7,0x96,0xd7,0x94,0x0d,0x0a,0xd7,0x90,
+
0xd7,0x97,0xd7,0xaa,0x0d,0x0a,0xd7,0x91,0xd7,0x91,0xd7,0x99,0xd7,0xaa,0x0d,0x0a,
+
0xd7,0x90,0xd7,0x9c,0xd7,0x94,0x0d,0x0a,0xd7,0x90,0xd7,0xa0,0xd7,0x97,0xd7,0xa0,
+ 0xd7,0x95,0x0d,0x0a
+ };
+
+static std::map<std::string, std::pair<const unsigned char*, unsigned int> >
createResourceMap() {
+ std::map<std::string, std::pair<const unsigned char*, unsigned int> > m;
+ m["stopwords/en"] = std::pair <const unsigned char*, unsigned
int>(stopwords_en, sizeof stopwords_en);
+ m["stopwords/fra"] = std::pair <const unsigned char*, unsigned
int>(stopwords_fra, sizeof stopwords_fra);
+ m["stopwords/he"] = std::pair <const unsigned char*, unsigned
int>(stopwords_he, sizeof stopwords_he);
+ return m;
+}
+
+static std::map<std::string, std::pair<const unsigned char*, unsigned int> >
resourceMap = createResourceMap();
+
+std::string getResourceAsString(const std::string &name) {
+ std::map<std::string, std::pair<const unsigned char*, unsigned int>
>::iterator it = resourceMap.find(name);
+ if (it != resourceMap.end()) {
+ return std::string((const char*)resourceMap[name].first,
resourceMap[name].second);
+ }
+ return "";
+}
diff --git a/zimwriterfs/resourceTools.h b/zimwriterfs/resourceTools.h
new file mode 100644
index 0000000..c040f86
--- /dev/null
+++ b/zimwriterfs/resourceTools.h
@@ -0,0 +1,10 @@
+#ifndef OPENZIM_ZIMWRITERFS_RESOURCETOOLS_H
+#define OPENZIM_ZIMWRITERFS_RESOURCETOOLS_H
+
+#include <string>
+
+std::string getResourceAsString(const std::string &name);
+
+
+#endif // OPENZIM_ZIMWRITERFS_RESOURCETOOLS_H
+
diff --git a/zimwriterfs/tools.cpp b/zimwriterfs/tools.cpp
index 019b22c..868f32c 100644
--- a/zimwriterfs/tools.cpp
+++ b/zimwriterfs/tools.cpp
@@ -32,6 +32,10 @@
#include <sys/stat.h>
#include <magic.h>
+#include <unicode/translit.h>
+#include <unicode/ucnv.h>
+
+
#ifdef _WIN32
#define SEPARATOR "\\"
#else
@@ -523,3 +527,14 @@
return computeRelativePath(baseUrl, newUrl);
}
+std::string removeAccents(const std::string &text) {
+ ucnv_setDefaultName("UTF-8");
+ UErrorCode status = U_ZERO_ERROR;
+ Transliterator *removeAccentsTrans = Transliterator::createInstance("Lower;
NFD; [:M:] remove; NFC", UTRANS_FORWARD, status);
+ UnicodeString ustring = UnicodeString(text.c_str());
+ removeAccentsTrans->transliterate(ustring);
+ delete removeAccentsTrans;
+ std::string unaccentedText;
+ ustring.toUTF8String(unaccentedText);
+ return unaccentedText;
+}
diff --git a/zimwriterfs/tools.h b/zimwriterfs/tools.h
index ec6b454..899cb65 100644
--- a/zimwriterfs/tools.h
+++ b/zimwriterfs/tools.h
@@ -44,4 +44,6 @@
std::string extractRedirectUrlFromHtml(const GumboVector* head_children);
void getLinks(GumboNode* node, std::map<std::string, bool> &links);
+std::string removeAccents(const std::string &text);
+
#endif // OPENZIM_ZIMWRITERFS_TOOLS_H
--
To view, visit https://gerrit.wikimedia.org/r/295520
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I0683706a136fb2303234e4caee77e9221714a5b1
Gerrit-PatchSet: 1
Gerrit-Project: openzim
Gerrit-Branch: master
Gerrit-Owner: Mgautierfr <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
