Kelson has submitted this change and it was merged. Change subject: Add a indexer. ......................................................................
Add a indexer. This indexer is not used. This is mainly code from kiwix-indexer imported in openzim. Unused function in *Tools has been removed. No dependency to xapian. Change-Id: I55079339d21d6903634c265f83f4d1c6ba0ac333 --- M zimwriterfs/Makefile.am A zimwriterfs/indexer.cpp A zimwriterfs/indexer.h A zimwriterfs/pathTools.cpp A zimwriterfs/pathTools.h A zimwriterfs/resourceTools.cpp A zimwriterfs/resourceTools.h M zimwriterfs/zimwriterfs.cpp 8 files changed, 921 insertions(+), 2 deletions(-) Approvals: Kelson: Verified; Looks good to me, approved diff --git a/zimwriterfs/Makefile.am b/zimwriterfs/Makefile.am index 92641d9..628b74c 100644 --- a/zimwriterfs/Makefile.am +++ b/zimwriterfs/Makefile.am @@ -6,4 +6,7 @@ tools.cpp \ article.cpp \ articlesource.cpp \ + indexer.cpp \ + resourceTools.cpp \ + pathTools.cpp \ mimetypecounter.cpp diff --git a/zimwriterfs/indexer.cpp b/zimwriterfs/indexer.cpp new file mode 100644 index 0000000..7820a32 --- /dev/null +++ b/zimwriterfs/indexer.cpp @@ -0,0 +1,262 @@ +/* + * Copyright 2011-2014 Emmanuel Engelhart <kel...@kiwix.org> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + * MA 02110-1301, USA. + */ + +#include "indexer.h" +#include "resourceTools.h" +#include "pathTools.h" +#include <unistd.h> + + /* Count word */ + unsigned int Indexer::countWords(const string &text) { + unsigned int numWords = 1; + unsigned int length = text.size(); + + for(unsigned int i=0; i<length;) { + while(i<length && text[i] != ' ') { + i++; + } + numWords++; + i++; + } + + return numWords; + } + + /* Constructor */ + Indexer::Indexer() : + keywordsBoostFactor(3), + verboseFlag(false) { + + /* Initialize mutex */ + pthread_mutex_init(&threadIdsMutex, NULL); + pthread_mutex_init(&toIndexQueueMutex, NULL); + pthread_mutex_init(&articleIndexerRunningMutex, NULL); + pthread_mutex_init(&articleCountMutex, NULL); + pthread_mutex_init(&zimIdMutex, NULL); + pthread_mutex_init(&indexPathMutex, NULL); + pthread_mutex_init(&verboseMutex, NULL); + } + + /* Destructor */ + Indexer::~Indexer() { + } + + /* Read the stopwords */ + void Indexer::readStopWords(const string languageCode) { + std::string stopWord; + std::istringstream file(getResourceAsString("stopwords/" + languageCode)); + + this->stopWords.clear(); + + while (getline(file, stopWord, '\n')) { + this->stopWords.push_back(stopWord); + } + } + + /* Article indexer methods */ + void *Indexer::indexArticles(void *ptr) { + pthread_setcanceltype(PTHREAD_CANCEL_DEFERRED, NULL); + Indexer *self = (Indexer *)ptr; + unsigned int indexedArticleCount = 0; + indexerToken token; + + self->indexingPrelude(self->getIndexPath()); + + while (self->popFromToIndexQueue(token)) { + self->index(token.url, + token.accentedTitle, + token.title, + token.keywords, + token.content, + token.snippet, + token.size, + token.wordCount + ); + + indexedArticleCount += 1; + + /* Make a hard-disk flush every 10.000 articles */ + if (indexedArticleCount % 5000 == 0) { + self->flush(); + } + + /* Test if the thread should be cancelled */ + pthread_testcancel(); + } + self->indexingPostlude(); + + /* Write content id file */ + string path = appendToDirectory(self->getIndexPath(), "content.id"); + writeTextFile(path, self->getZimId()); + + usleep(100); + + self->articleIndexerRunning(false); + pthread_exit(NULL); + return NULL; + } + + void Indexer::articleIndexerRunning(bool value) { + pthread_mutex_lock(&articleIndexerRunningMutex); + this->articleIndexerRunningFlag = value; + pthread_mutex_unlock(&articleIndexerRunningMutex); + } + + bool Indexer::isArticleIndexerRunning() { + pthread_mutex_lock(&articleIndexerRunningMutex); + bool retVal = this->articleIndexerRunningFlag; + pthread_mutex_unlock(&articleIndexerRunningMutex); + return retVal; + } + + /* ToIndexQueue methods */ + bool Indexer::isToIndexQueueEmpty() { + pthread_mutex_lock(&toIndexQueueMutex); + bool retVal = this->toIndexQueue.empty(); + pthread_mutex_unlock(&toIndexQueueMutex); + return retVal; + } + + void Indexer::pushToIndexQueue(indexerToken &token) { + pthread_mutex_lock(&toIndexQueueMutex); + this->toIndexQueue.push(token); + pthread_mutex_unlock(&toIndexQueueMutex); + usleep(int(this->toIndexQueue.size() / 200) / 10 * 1000); + } + + bool Indexer::popFromToIndexQueue(indexerToken &token) { + while (this->isToIndexQueueEmpty()) { + usleep(500); + if (this->getVerboseFlag()) { + std::cout << "Waiting... ToIndexQueue is empty for now..." << std::endl; + } + + pthread_testcancel(); + } + + pthread_mutex_lock(&toIndexQueueMutex); + token = this->toIndexQueue.front(); + this->toIndexQueue.pop(); + pthread_mutex_unlock(&toIndexQueueMutex); + + if (token.title == ""){ + //This is a empty token, end of the queue. + return false; + } + return true; + } + + /* Index methods */ + void Indexer::setIndexPath(const string path) { + pthread_mutex_lock(&indexPathMutex); + this->indexPath = path; + pthread_mutex_unlock(&indexPathMutex); + } + + string Indexer::getIndexPath() { + pthread_mutex_lock(&indexPathMutex); + string retVal = this->indexPath; + pthread_mutex_unlock(&indexPathMutex); + return retVal; + } + + void Indexer::setArticleCount(const unsigned int articleCount) { + pthread_mutex_lock(&articleCountMutex); + this->articleCount = articleCount; + pthread_mutex_unlock(&articleCountMutex); + } + + unsigned int Indexer::getArticleCount() { + pthread_mutex_lock(&articleCountMutex); + unsigned int retVal = this->articleCount; + pthread_mutex_unlock(&articleCountMutex); + return retVal; + } + + void Indexer::setZimId(const string id) { + pthread_mutex_lock(&zimIdMutex); + this->zimId = id; + pthread_mutex_unlock(&zimIdMutex); + } + + string Indexer::getZimId() { + pthread_mutex_lock(&zimIdMutex); + string retVal = this->zimId; + pthread_mutex_unlock(&zimIdMutex); + return retVal; + } + + /* Manage */ + bool Indexer::start(const string indexPath) { + if (this->getVerboseFlag()) { + std::cout << "Indexing starting..." <<std::endl; + } + + this->setArticleCount(0); + this->setIndexPath(indexPath); + + pthread_mutex_lock(&threadIdsMutex); + + this->articleIndexerRunning(true); + pthread_create(&(this->articleIndexer), NULL, Indexer::indexArticles, (void*)this); + pthread_detach(this->articleIndexer); + pthread_mutex_unlock(&threadIdsMutex); + + return true; + } + + bool Indexer::isRunning() { + if (this->getVerboseFlag()) { + std::cout << "isArticleIndexer running: " << (this->isArticleIndexerRunning() ? "yes" : "no") << std::endl; + } + + return this->isArticleIndexerRunning(); + } + + bool Indexer::stop() { + if (this->isRunning()) { + bool isArticleIndexerRunning = this->isArticleIndexerRunning(); + + pthread_mutex_lock(&threadIdsMutex); + + if (isArticleIndexerRunning) { + pthread_cancel(this->articleIndexer); + this->articleIndexerRunning(false); + } + + pthread_mutex_unlock(&threadIdsMutex); + } + + return true; + } + + /* Manage the verboseFlag */ + void Indexer::setVerboseFlag(const bool value) { + pthread_mutex_lock(&verboseMutex); + this->verboseFlag = value; + pthread_mutex_unlock(&verboseMutex); + } + + bool Indexer::getVerboseFlag() { + bool value; + pthread_mutex_lock(&verboseMutex); + value = this->verboseFlag; + pthread_mutex_unlock(&verboseMutex); + return value; + } diff --git a/zimwriterfs/indexer.h b/zimwriterfs/indexer.h new file mode 100644 index 0000000..02d989b --- /dev/null +++ b/zimwriterfs/indexer.h @@ -0,0 +1,134 @@ +/* + * Copyright 2014 Emmanuel Engelhart <kel...@kiwix.org> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + * MA 02110-1301, USA. + */ + +#ifndef OPENZIM_ZIMWRITERFS_INDEXER_H +#define OPENZIM_ZIMWRITERFS_INDEXER_H + +#include <string> +#include <vector> +#include <stack> +#include <queue> +#include <fstream> +#include <iostream> +#include <sstream> + +#include <pthread.h> +/*#include <stringTools.h> +#include <otherTools.h> +#include <resourceTools.h>*/ +#include <zim/file.h> +#include <zim/article.h> +#include <zim/fileiterator.h> +#include <zim/writer/zimcreator.h> +/*#include "reader.h"*/ + +using namespace std; + +struct indexerToken { + string url; + string accentedTitle; + string title; + string keywords; + string content; + string snippet; + string size; + string wordCount; +}; + +class Indexer { + + public: + Indexer(); + virtual ~Indexer(); + + bool start(const string indexPath); + bool stop(); + bool isRunning(); + void setVerboseFlag(const bool value); + void pushToIndexQueue(indexerToken &token); + + protected: + virtual void indexingPrelude(const string indexPath) = 0; + virtual void index(const string &url, + const string &title, + const string &unaccentedTitle, + const string &keywords, + const string &content, + const string &snippet, + const string &size, + const string &wordCount) = 0; + virtual void flush() = 0; + virtual void indexingPostlude() = 0; + + /* Stop words */ + std::vector<std::string> stopWords; + void readStopWords(const string languageCode); + + /* Others */ + unsigned int countWords(const string &text); + + /* Boost factor */ + unsigned int keywordsBoostFactor; + inline unsigned int getTitleBoostFactor(const unsigned int contentLength) { + return contentLength / 500 + 1; + } + + /* Verbose */ + pthread_mutex_t verboseMutex; + bool getVerboseFlag(); + bool verboseFlag; + + private: + pthread_mutex_t threadIdsMutex; + + /* Index writting */ + pthread_t articleIndexer; + pthread_mutex_t articleIndexerRunningMutex; + static void *indexArticles(void *ptr); + bool articleIndexerRunningFlag; + bool isArticleIndexerRunning(); + void articleIndexerRunning(bool value); + + /* To index queue */ + std::queue<indexerToken> toIndexQueue; + pthread_mutex_t toIndexQueueMutex; + /* void pushToIndexQueue(indexerToken &token); is public */ + bool popFromToIndexQueue(indexerToken &token); + bool isToIndexQueueEmpty(); + + /* Article Count & Progression */ + unsigned int articleCount; + pthread_mutex_t articleCountMutex; + void setArticleCount(const unsigned int articleCount); + unsigned int getArticleCount(); + + /* Index path */ + pthread_mutex_t indexPathMutex; + string indexPath; + void setIndexPath(const string path); + string getIndexPath(); + + /* ZIM id */ + pthread_mutex_t zimIdMutex; + string zimId; + void setZimId(const string id); + string getZimId(); + }; + +#endif //OPENZIM_ZIMWRITERFS_INDEXER_H diff --git a/zimwriterfs/pathTools.cpp b/zimwriterfs/pathTools.cpp new file mode 100644 index 0000000..3570252 --- /dev/null +++ b/zimwriterfs/pathTools.cpp @@ -0,0 +1,78 @@ +/* + * Copyright 2011-2014 Emmanuel Engelhart <kel...@kiwix.org> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + * MA 02110-1301, USA. + */ + +#include "pathTools.h" + +#ifdef __APPLE__ +#include <mach-o/dyld.h> +#include <limits.h> +#elif _WIN32 +#include <windows.h> +#include "Shlwapi.h" +#endif + +#ifdef _WIN32 +#else +#include <unistd.h> +#endif + +#ifdef _WIN32 +#define SEPARATOR "\\" +#else +#define SEPARATOR "/" +#endif + +#ifndef PATH_MAX +#define PATH_MAX 1024 +#endif + + +std::string appendToDirectory(const std::string &directoryPath, const std::string &filename) { + std::string newPath = directoryPath + SEPARATOR + filename; + return newPath; +} + + +std::string getExecutablePath() { + char binRootPath[PATH_MAX]; + +#ifdef _WIN32 + GetModuleFileName( NULL, binRootPath, PATH_MAX); + return std::string(binRootPath); +#elif __APPLE__ + uint32_t max = (uint32_t)PATH_MAX; + _NSGetExecutablePath(binRootPath, &max); + return std::string(binRootPath); +#else + ssize_t size = readlink("/proc/self/exe", binRootPath, PATH_MAX); + if (size != -1) { + return std::string(binRootPath, size); + } +#endif + + return ""; +} + +bool writeTextFile(const std::string &path, const std::string &content) { + std::ofstream file; + file.open(path.c_str()); + file << content; + file.close(); + return true; +} diff --git a/zimwriterfs/pathTools.h b/zimwriterfs/pathTools.h new file mode 100644 index 0000000..b66486e --- /dev/null +++ b/zimwriterfs/pathTools.h @@ -0,0 +1,46 @@ +/* + * Copyright 2011 Emmanuel Engelhart <kel...@kiwix.org> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + * MA 02110-1301, USA. + */ + +#ifndef OPENZIM_ZIMWRITERFS_PATHTOOLS_H +#define OPENZIM_ZIMWRITERFS_PATHTOOLS_H + +#include <string> +#include <vector> +#include <sstream> +#include <iostream> +#include <fstream> +#include <string.h> +#include <stdio.h> +#include <sys/types.h> +#include <fcntl.h> +#include <sys/stat.h> +#include <ios> +#include <limits.h> + +#ifdef _WIN32 +#include <direct.h> +#endif + +std::string appendToDirectory(const std::string &directoryPath, const std::string &filename); + +std::string getExecutablePath(); + +bool writeTextFile(const std::string &path, const std::string &content); + +#endif //OPENZIM_ZIMWRITERFS_PATHTOOLS_H diff --git a/zimwriterfs/resourceTools.cpp b/zimwriterfs/resourceTools.cpp new file mode 100644 index 0000000..c00e5a0 --- /dev/null +++ b/zimwriterfs/resourceTools.cpp @@ -0,0 +1,380 @@ + +#include <resourceTools.h> +#include <string> +#include <map> + +static const unsigned char stopwords_en[]={ + 0x61,0x0a,0x61,0x62,0x6c,0x65,0x0a,0x61,0x62,0x6f,0x75,0x74,0x0a,0x61,0x62,0x6f, + 0x76,0x65,0x0a,0x61,0x62,0x73,0x74,0x0a,0x61,0x63,0x63,0x6f,0x72,0x64,0x61,0x6e, + 0x63,0x65,0x0a,0x61,0x63,0x63,0x6f,0x72,0x64,0x69,0x6e,0x67,0x0a,0x61,0x63,0x63, + 0x6f,0x72,0x64,0x69,0x6e,0x67,0x6c,0x79,0x0a,0x61,0x63,0x72,0x6f,0x73,0x73,0x0a, + 0x61,0x63,0x74,0x0a,0x61,0x63,0x74,0x75,0x61,0x6c,0x6c,0x79,0x0a,0x61,0x64,0x64, + 0x65,0x64,0x0a,0x61,0x64,0x6a,0x0a,0x61,0x64,0x6f,0x70,0x74,0x65,0x64,0x0a,0x61, + 0x66,0x66,0x65,0x63,0x74,0x65,0x64,0x0a,0x61,0x66,0x66,0x65,0x63,0x74,0x69,0x6e, + 0x67,0x0a,0x61,0x66,0x66,0x65,0x63,0x74,0x73,0x0a,0x61,0x66,0x74,0x65,0x72,0x0a, + 0x61,0x66,0x74,0x65,0x72,0x77,0x61,0x72,0x64,0x73,0x0a,0x61,0x67,0x61,0x69,0x6e, + 0x0a,0x61,0x67,0x61,0x69,0x6e,0x73,0x74,0x0a,0x61,0x68,0x0a,0x61,0x6c,0x6c,0x0a, + 0x61,0x6c,0x6d,0x6f,0x73,0x74,0x0a,0x61,0x6c,0x6f,0x6e,0x65,0x0a,0x61,0x6c,0x6f, + 0x6e,0x67,0x0a,0x61,0x6c,0x72,0x65,0x61,0x64,0x79,0x0a,0x61,0x6c,0x73,0x6f,0x0a, + 0x61,0x6c,0x74,0x68,0x6f,0x75,0x67,0x68,0x0a,0x61,0x6c,0x77,0x61,0x79,0x73,0x0a, + 0x61,0x6d,0x0a,0x61,0x6d,0x6f,0x6e,0x67,0x0a,0x61,0x6d,0x6f,0x6e,0x67,0x73,0x74, + 0x0a,0x61,0x6e,0x0a,0x61,0x6e,0x64,0x0a,0x61,0x6e,0x6e,0x6f,0x75,0x6e,0x63,0x65, + 0x0a,0x61,0x6e,0x6f,0x74,0x68,0x65,0x72,0x0a,0x61,0x6e,0x79,0x0a,0x61,0x6e,0x79, + 0x62,0x6f,0x64,0x79,0x0a,0x61,0x6e,0x79,0x68,0x6f,0x77,0x0a,0x61,0x6e,0x79,0x6d, + 0x6f,0x72,0x65,0x0a,0x61,0x6e,0x79,0x6f,0x6e,0x65,0x0a,0x61,0x6e,0x79,0x74,0x68, + 0x69,0x6e,0x67,0x0a,0x61,0x6e,0x79,0x77,0x61,0x79,0x0a,0x61,0x6e,0x79,0x77,0x61, + 0x79,0x73,0x0a,0x61,0x6e,0x79,0x77,0x68,0x65,0x72,0x65,0x0a,0x61,0x70,0x70,0x61, + 0x72,0x65,0x6e,0x74,0x6c,0x79,0x0a,0x61,0x70,0x70,0x72,0x6f,0x78,0x69,0x6d,0x61, + 0x74,0x65,0x6c,0x79,0x0a,0x61,0x72,0x65,0x0a,0x61,0x72,0x65,0x6e,0x0a,0x61,0x72, + 0x65,0x6e,0x74,0x0a,0x61,0x72,0x69,0x73,0x65,0x0a,0x61,0x72,0x6f,0x75,0x6e,0x64, + 0x0a,0x61,0x73,0x0a,0x61,0x73,0x69,0x64,0x65,0x0a,0x61,0x73,0x6b,0x0a,0x61,0x73, + 0x6b,0x69,0x6e,0x67,0x0a,0x61,0x74,0x0a,0x61,0x75,0x74,0x68,0x0a,0x61,0x76,0x61, + 0x69,0x6c,0x61,0x62,0x6c,0x65,0x0a,0x61,0x77,0x61,0x79,0x0a,0x61,0x77,0x66,0x75, + 0x6c,0x6c,0x79,0x0a,0x62,0x0a,0x62,0x61,0x63,0x6b,0x0a,0x62,0x65,0x0a,0x62,0x65, + 0x63,0x61,0x6d,0x65,0x0a,0x62,0x65,0x63,0x61,0x75,0x73,0x65,0x0a,0x62,0x65,0x63, + 0x6f,0x6d,0x65,0x0a,0x62,0x65,0x63,0x6f,0x6d,0x65,0x73,0x0a,0x62,0x65,0x63,0x6f, + 0x6d,0x69,0x6e,0x67,0x0a,0x62,0x65,0x65,0x6e,0x0a,0x62,0x65,0x66,0x6f,0x72,0x65, + 0x0a,0x62,0x65,0x66,0x6f,0x72,0x65,0x68,0x61,0x6e,0x64,0x0a,0x62,0x65,0x67,0x69, + 0x6e,0x0a,0x62,0x65,0x67,0x69,0x6e,0x6e,0x69,0x6e,0x67,0x0a,0x62,0x65,0x67,0x69, + 0x6e,0x6e,0x69,0x6e,0x67,0x73,0x0a,0x62,0x65,0x67,0x69,0x6e,0x73,0x0a,0x62,0x65, + 0x68,0x69,0x6e,0x64,0x0a,0x62,0x65,0x69,0x6e,0x67,0x0a,0x62,0x65,0x6c,0x69,0x65, + 0x76,0x65,0x0a,0x62,0x65,0x6c,0x6f,0x77,0x0a,0x62,0x65,0x73,0x69,0x64,0x65,0x0a, + 0x62,0x65,0x73,0x69,0x64,0x65,0x73,0x0a,0x62,0x65,0x74,0x77,0x65,0x65,0x6e,0x0a, + 0x62,0x65,0x79,0x6f,0x6e,0x64,0x0a,0x62,0x69,0x6f,0x6c,0x0a,0x62,0x6f,0x74,0x68, + 0x0a,0x62,0x72,0x69,0x65,0x66,0x0a,0x62,0x72,0x69,0x65,0x66,0x6c,0x79,0x0a,0x62, + 0x75,0x74,0x0a,0x62,0x79,0x0a,0x63,0x0a,0x63,0x61,0x0a,0x63,0x61,0x6d,0x65,0x0a, + 0x63,0x61,0x6e,0x0a,0x63,0x61,0x6e,0x6e,0x6f,0x74,0x0a,0x63,0x61,0x6e,0x27,0x74, + 0x0a,0x63,0x61,0x75,0x73,0x65,0x0a,0x63,0x61,0x75,0x73,0x65,0x73,0x0a,0x63,0x65, + 0x72,0x74,0x61,0x69,0x6e,0x0a,0x63,0x65,0x72,0x74,0x61,0x69,0x6e,0x6c,0x79,0x0a, + 0x63,0x6f,0x0a,0x63,0x6f,0x6d,0x0a,0x63,0x6f,0x6d,0x65,0x0a,0x63,0x6f,0x6d,0x65, + 0x73,0x0a,0x63,0x6f,0x6e,0x74,0x61,0x69,0x6e,0x0a,0x63,0x6f,0x6e,0x74,0x61,0x69, + 0x6e,0x69,0x6e,0x67,0x0a,0x63,0x6f,0x6e,0x74,0x61,0x69,0x6e,0x73,0x0a,0x63,0x6f, + 0x75,0x6c,0x64,0x0a,0x63,0x6f,0x75,0x6c,0x64,0x6e,0x74,0x0a,0x64,0x0a,0x64,0x61, + 0x74,0x65,0x0a,0x64,0x69,0x64,0x0a,0x64,0x69,0x64,0x6e,0x27,0x74,0x0a,0x64,0x69, + 0x66,0x66,0x65,0x72,0x65,0x6e,0x74,0x0a,0x64,0x6f,0x0a,0x64,0x6f,0x65,0x73,0x0a, + 0x64,0x6f,0x65,0x73,0x6e,0x27,0x74,0x0a,0x64,0x6f,0x69,0x6e,0x67,0x0a,0x64,0x6f, + 0x6e,0x65,0x0a,0x64,0x6f,0x6e,0x27,0x74,0x0a,0x64,0x6f,0x77,0x6e,0x0a,0x64,0x6f, + 0x77,0x6e,0x77,0x61,0x72,0x64,0x73,0x0a,0x64,0x75,0x65,0x0a,0x64,0x75,0x72,0x69, + 0x6e,0x67,0x0a,0x65,0x0a,0x65,0x61,0x63,0x68,0x0a,0x65,0x64,0x0a,0x65,0x64,0x75, + 0x0a,0x65,0x66,0x66,0x65,0x63,0x74,0x0a,0x65,0x67,0x0a,0x65,0x69,0x67,0x68,0x74, + 0x0a,0x65,0x69,0x67,0x68,0x74,0x79,0x0a,0x65,0x69,0x74,0x68,0x65,0x72,0x0a,0x65, + 0x6c,0x73,0x65,0x0a,0x65,0x6c,0x73,0x65,0x77,0x68,0x65,0x72,0x65,0x0a,0x65,0x6e, + 0x64,0x0a,0x65,0x6e,0x64,0x69,0x6e,0x67,0x0a,0x65,0x6e,0x6f,0x75,0x67,0x68,0x0a, + 0x65,0x73,0x70,0x65,0x63,0x69,0x61,0x6c,0x6c,0x79,0x0a,0x65,0x74,0x0a,0x65,0x74, + 0x2d,0x61,0x6c,0x0a,0x65,0x74,0x63,0x0a,0x65,0x76,0x65,0x6e,0x0a,0x65,0x76,0x65, + 0x72,0x0a,0x65,0x76,0x65,0x72,0x79,0x0a,0x65,0x76,0x65,0x72,0x79,0x62,0x6f,0x64, + 0x79,0x0a,0x65,0x76,0x65,0x72,0x79,0x6f,0x6e,0x65,0x0a,0x65,0x76,0x65,0x72,0x79, + 0x74,0x68,0x69,0x6e,0x67,0x0a,0x65,0x76,0x65,0x72,0x79,0x77,0x68,0x65,0x72,0x65, + 0x0a,0x65,0x78,0x0a,0x65,0x78,0x63,0x65,0x70,0x74,0x0a,0x66,0x0a,0x66,0x61,0x72, + 0x0a,0x66,0x65,0x77,0x0a,0x66,0x66,0x0a,0x66,0x69,0x66,0x74,0x68,0x0a,0x66,0x69, + 0x72,0x73,0x74,0x0a,0x66,0x69,0x76,0x65,0x0a,0x66,0x69,0x78,0x0a,0x66,0x6f,0x6c, + 0x6c,0x6f,0x77,0x65,0x64,0x0a,0x66,0x6f,0x6c,0x6c,0x6f,0x77,0x69,0x6e,0x67,0x0a, + 0x66,0x6f,0x6c,0x6c,0x6f,0x77,0x73,0x0a,0x66,0x6f,0x72,0x0a,0x66,0x6f,0x72,0x6d, + 0x65,0x72,0x0a,0x66,0x6f,0x72,0x6d,0x65,0x72,0x6c,0x79,0x0a,0x66,0x6f,0x72,0x74, + 0x68,0x0a,0x66,0x6f,0x75,0x6e,0x64,0x0a,0x66,0x6f,0x75,0x72,0x0a,0x66,0x72,0x6f, + 0x6d,0x0a,0x66,0x75,0x72,0x74,0x68,0x65,0x72,0x0a,0x66,0x75,0x72,0x74,0x68,0x65, + 0x72,0x6d,0x6f,0x72,0x65,0x0a,0x67,0x0a,0x67,0x61,0x76,0x65,0x0a,0x67,0x65,0x74, + 0x0a,0x67,0x65,0x74,0x73,0x0a,0x67,0x65,0x74,0x74,0x69,0x6e,0x67,0x0a,0x67,0x69, + 0x76,0x65,0x0a,0x67,0x69,0x76,0x65,0x6e,0x0a,0x67,0x69,0x76,0x65,0x73,0x0a,0x67, + 0x69,0x76,0x69,0x6e,0x67,0x0a,0x67,0x6f,0x0a,0x67,0x6f,0x65,0x73,0x0a,0x67,0x6f, + 0x6e,0x65,0x0a,0x67,0x6f,0x74,0x0a,0x67,0x6f,0x74,0x74,0x65,0x6e,0x0a,0x68,0x0a, + 0x68,0x61,0x64,0x0a,0x68,0x61,0x70,0x70,0x65,0x6e,0x73,0x0a,0x68,0x61,0x72,0x64, + 0x6c,0x79,0x0a,0x68,0x61,0x73,0x0a,0x68,0x61,0x73,0x6e,0x27,0x74,0x0a,0x68,0x61, + 0x76,0x65,0x0a,0x68,0x61,0x76,0x65,0x6e,0x27,0x74,0x0a,0x68,0x61,0x76,0x69,0x6e, + 0x67,0x0a,0x68,0x65,0x0a,0x68,0x65,0x64,0x0a,0x68,0x65,0x6e,0x63,0x65,0x0a,0x68, + 0x65,0x72,0x0a,0x68,0x65,0x72,0x65,0x0a,0x68,0x65,0x72,0x65,0x61,0x66,0x74,0x65, + 0x72,0x0a,0x68,0x65,0x72,0x65,0x62,0x79,0x0a,0x68,0x65,0x72,0x65,0x69,0x6e,0x0a, + 0x68,0x65,0x72,0x65,0x73,0x0a,0x68,0x65,0x72,0x65,0x75,0x70,0x6f,0x6e,0x0a,0x68, + 0x65,0x72,0x73,0x0a,0x68,0x65,0x72,0x73,0x65,0x6c,0x66,0x0a,0x68,0x65,0x73,0x0a, + 0x68,0x69,0x0a,0x68,0x69,0x64,0x0a,0x68,0x69,0x6d,0x0a,0x68,0x69,0x6d,0x73,0x65, + 0x6c,0x66,0x0a,0x68,0x69,0x73,0x0a,0x68,0x69,0x74,0x68,0x65,0x72,0x0a,0x68,0x6f, + 0x6d,0x65,0x0a,0x68,0x6f,0x77,0x0a,0x68,0x6f,0x77,0x62,0x65,0x69,0x74,0x0a,0x68, + 0x6f,0x77,0x65,0x76,0x65,0x72,0x0a,0x68,0x75,0x6e,0x64,0x72,0x65,0x64,0x0a,0x69, + 0x0a,0x69,0x64,0x0a,0x69,0x65,0x0a,0x69,0x66,0x0a,0x69,0x27,0x6c,0x6c,0x0a,0x69, + 0x6d,0x0a,0x69,0x6d,0x6d,0x65,0x64,0x69,0x61,0x74,0x65,0x0a,0x69,0x6d,0x6d,0x65, + 0x64,0x69,0x61,0x74,0x65,0x6c,0x79,0x0a,0x69,0x6d,0x70,0x6f,0x72,0x74,0x61,0x6e, + 0x63,0x65,0x0a,0x69,0x6d,0x70,0x6f,0x72,0x74,0x61,0x6e,0x74,0x0a,0x69,0x6e,0x0a, + 0x69,0x6e,0x63,0x0a,0x69,0x6e,0x64,0x65,0x65,0x64,0x0a,0x69,0x6e,0x64,0x65,0x78, + 0x0a,0x69,0x6e,0x66,0x6f,0x72,0x6d,0x61,0x74,0x69,0x6f,0x6e,0x0a,0x69,0x6e,0x73, + 0x74,0x65,0x61,0x64,0x0a,0x69,0x6e,0x74,0x6f,0x0a,0x69,0x6e,0x76,0x65,0x6e,0x74, + 0x69,0x6f,0x6e,0x0a,0x69,0x6e,0x77,0x61,0x72,0x64,0x0a,0x69,0x73,0x0a,0x69,0x73, + 0x6e,0x27,0x74,0x0a,0x69,0x74,0x0a,0x69,0x74,0x64,0x0a,0x69,0x74,0x27,0x6c,0x6c, + 0x0a,0x69,0x74,0x73,0x0a,0x69,0x74,0x73,0x65,0x6c,0x66,0x0a,0x69,0x27,0x76,0x65, + 0x0a,0x6a,0x0a,0x6a,0x75,0x73,0x74,0x0a,0x6b,0x0a,0x6b,0x65,0x65,0x70,0x0a,0x6b, + 0x65,0x65,0x70,0x73,0x0a,0x6b,0x65,0x70,0x74,0x0a,0x6b,0x65,0x79,0x73,0x0a,0x6b, + 0x67,0x0a,0x6b,0x6d,0x0a,0x6b,0x6e,0x6f,0x77,0x0a,0x6b,0x6e,0x6f,0x77,0x6e,0x0a, + 0x6b,0x6e,0x6f,0x77,0x73,0x0a,0x6c,0x0a,0x6c,0x61,0x72,0x67,0x65,0x6c,0x79,0x0a, + 0x6c,0x61,0x73,0x74,0x0a,0x6c,0x61,0x74,0x65,0x6c,0x79,0x0a,0x6c,0x61,0x74,0x65, + 0x72,0x0a,0x6c,0x61,0x74,0x74,0x65,0x72,0x0a,0x6c,0x61,0x74,0x74,0x65,0x72,0x6c, + 0x79,0x0a,0x6c,0x65,0x61,0x73,0x74,0x0a,0x6c,0x65,0x73,0x73,0x0a,0x6c,0x65,0x73, + 0x74,0x0a,0x6c,0x65,0x74,0x0a,0x6c,0x65,0x74,0x73,0x0a,0x6c,0x69,0x6b,0x65,0x0a, + 0x6c,0x69,0x6b,0x65,0x64,0x0a,0x6c,0x69,0x6b,0x65,0x6c,0x79,0x0a,0x6c,0x69,0x6e, + 0x65,0x0a,0x6c,0x69,0x74,0x74,0x6c,0x65,0x0a,0x27,0x6c,0x6c,0x0a,0x6c,0x6f,0x6f, + 0x6b,0x0a,0x6c,0x6f,0x6f,0x6b,0x69,0x6e,0x67,0x0a,0x6c,0x6f,0x6f,0x6b,0x73,0x0a, + 0x6c,0x74,0x64,0x0a,0x6d,0x0a,0x6d,0x61,0x64,0x65,0x0a,0x6d,0x61,0x69,0x6e,0x6c, + 0x79,0x0a,0x6d,0x61,0x6b,0x65,0x0a,0x6d,0x61,0x6b,0x65,0x73,0x0a,0x6d,0x61,0x6e, + 0x79,0x0a,0x6d,0x61,0x79,0x0a,0x6d,0x61,0x79,0x62,0x65,0x0a,0x6d,0x65,0x0a,0x6d, + 0x65,0x61,0x6e,0x0a,0x6d,0x65,0x61,0x6e,0x73,0x0a,0x6d,0x65,0x61,0x6e,0x74,0x69, + 0x6d,0x65,0x0a,0x6d,0x65,0x61,0x6e,0x77,0x68,0x69,0x6c,0x65,0x0a,0x6d,0x65,0x72, + 0x65,0x6c,0x79,0x0a,0x6d,0x67,0x0a,0x6d,0x69,0x67,0x68,0x74,0x0a,0x6d,0x69,0x6c, + 0x6c,0x69,0x6f,0x6e,0x0a,0x6d,0x69,0x73,0x73,0x0a,0x6d,0x6c,0x0a,0x6d,0x6f,0x72, + 0x65,0x0a,0x6d,0x6f,0x72,0x65,0x6f,0x76,0x65,0x72,0x0a,0x6d,0x6f,0x73,0x74,0x0a, + 0x6d,0x6f,0x73,0x74,0x6c,0x79,0x0a,0x6d,0x72,0x0a,0x6d,0x72,0x73,0x0a,0x6d,0x75, + 0x63,0x68,0x0a,0x6d,0x75,0x67,0x0a,0x6d,0x75,0x73,0x74,0x0a,0x6d,0x79,0x0a,0x6d, + 0x79,0x73,0x65,0x6c,0x66,0x0a,0x6e,0x0a,0x6e,0x61,0x0a,0x6e,0x61,0x6d,0x65,0x0a, + 0x6e,0x61,0x6d,0x65,0x6c,0x79,0x0a,0x6e,0x61,0x79,0x0a,0x6e,0x64,0x0a,0x6e,0x65, + 0x61,0x72,0x0a,0x6e,0x65,0x61,0x72,0x6c,0x79,0x0a,0x6e,0x65,0x63,0x65,0x73,0x73, + 0x61,0x72,0x69,0x6c,0x79,0x0a,0x6e,0x65,0x63,0x65,0x73,0x73,0x61,0x72,0x79,0x0a, + 0x6e,0x65,0x65,0x64,0x0a,0x6e,0x65,0x65,0x64,0x73,0x0a,0x6e,0x65,0x69,0x74,0x68, + 0x65,0x72,0x0a,0x6e,0x65,0x76,0x65,0x72,0x0a,0x6e,0x65,0x76,0x65,0x72,0x74,0x68, + 0x65,0x6c,0x65,0x73,0x73,0x0a,0x6e,0x65,0x77,0x0a,0x6e,0x65,0x78,0x74,0x0a,0x6e, + 0x69,0x6e,0x65,0x0a,0x6e,0x69,0x6e,0x65,0x74,0x79,0x0a,0x6e,0x6f,0x0a,0x6e,0x6f, + 0x62,0x6f,0x64,0x79,0x0a,0x6e,0x6f,0x6e,0x0a,0x6e,0x6f,0x6e,0x65,0x0a,0x6e,0x6f, + 0x6e,0x65,0x74,0x68,0x65,0x6c,0x65,0x73,0x73,0x0a,0x6e,0x6f,0x6f,0x6e,0x65,0x0a, + 0x6e,0x6f,0x72,0x0a,0x6e,0x6f,0x72,0x6d,0x61,0x6c,0x6c,0x79,0x0a,0x6e,0x6f,0x73, + 0x0a,0x6e,0x6f,0x74,0x0a,0x6e,0x6f,0x74,0x65,0x64,0x0a,0x6e,0x6f,0x74,0x68,0x69, + 0x6e,0x67,0x0a,0x6e,0x6f,0x77,0x0a,0x6e,0x6f,0x77,0x68,0x65,0x72,0x65,0x0a,0x6f, + 0x0a,0x6f,0x62,0x74,0x61,0x69,0x6e,0x0a,0x6f,0x62,0x74,0x61,0x69,0x6e,0x65,0x64, + 0x0a,0x6f,0x62,0x76,0x69,0x6f,0x75,0x73,0x6c,0x79,0x0a,0x6f,0x66,0x0a,0x6f,0x66, + 0x66,0x0a,0x6f,0x66,0x74,0x65,0x6e,0x0a,0x6f,0x68,0x0a,0x6f,0x6b,0x0a,0x6f,0x6b, + 0x61,0x79,0x0a,0x6f,0x6c,0x64,0x0a,0x6f,0x6d,0x69,0x74,0x74,0x65,0x64,0x0a,0x6f, + 0x6e,0x0a,0x6f,0x6e,0x63,0x65,0x0a,0x6f,0x6e,0x65,0x0a,0x6f,0x6e,0x65,0x73,0x0a, + 0x6f,0x6e,0x6c,0x79,0x0a,0x6f,0x6e,0x74,0x6f,0x0a,0x6f,0x72,0x0a,0x6f,0x72,0x64, + 0x0a,0x6f,0x74,0x68,0x65,0x72,0x0a,0x6f,0x74,0x68,0x65,0x72,0x73,0x0a,0x6f,0x74, + 0x68,0x65,0x72,0x77,0x69,0x73,0x65,0x0a,0x6f,0x75,0x67,0x68,0x74,0x0a,0x6f,0x75, + 0x72,0x0a,0x6f,0x75,0x72,0x73,0x0a,0x6f,0x75,0x72,0x73,0x65,0x6c,0x76,0x65,0x73, + 0x0a,0x6f,0x75,0x74,0x0a,0x6f,0x75,0x74,0x73,0x69,0x64,0x65,0x0a,0x6f,0x76,0x65, + 0x72,0x0a,0x6f,0x76,0x65,0x72,0x61,0x6c,0x6c,0x0a,0x6f,0x77,0x69,0x6e,0x67,0x0a, + 0x6f,0x77,0x6e,0x0a,0x70,0x0a,0x70,0x61,0x67,0x65,0x0a,0x70,0x61,0x67,0x65,0x73, + 0x0a,0x70,0x61,0x72,0x74,0x0a,0x70,0x61,0x72,0x74,0x69,0x63,0x75,0x6c,0x61,0x72, + 0x0a,0x70,0x61,0x72,0x74,0x69,0x63,0x75,0x6c,0x61,0x72,0x6c,0x79,0x0a,0x70,0x61, + 0x73,0x74,0x0a,0x70,0x65,0x72,0x0a,0x70,0x65,0x72,0x68,0x61,0x70,0x73,0x0a,0x70, + 0x6c,0x61,0x63,0x65,0x64,0x0a,0x70,0x6c,0x65,0x61,0x73,0x65,0x0a,0x70,0x6c,0x75, + 0x73,0x0a,0x70,0x6f,0x6f,0x72,0x6c,0x79,0x0a,0x70,0x6f,0x73,0x73,0x69,0x62,0x6c, + 0x65,0x0a,0x70,0x6f,0x73,0x73,0x69,0x62,0x6c,0x79,0x0a,0x70,0x6f,0x74,0x65,0x6e, + 0x74,0x69,0x61,0x6c,0x6c,0x79,0x0a,0x70,0x70,0x0a,0x70,0x72,0x65,0x64,0x6f,0x6d, + 0x69,0x6e,0x61,0x6e,0x74,0x6c,0x79,0x0a,0x70,0x72,0x65,0x73,0x65,0x6e,0x74,0x0a, + 0x70,0x72,0x65,0x76,0x69,0x6f,0x75,0x73,0x6c,0x79,0x0a,0x70,0x72,0x69,0x6d,0x61, + 0x72,0x69,0x6c,0x79,0x0a,0x70,0x72,0x6f,0x62,0x61,0x62,0x6c,0x79,0x0a,0x70,0x72, + 0x6f,0x6d,0x70,0x74,0x6c,0x79,0x0a,0x70,0x72,0x6f,0x75,0x64,0x0a,0x70,0x72,0x6f, + 0x76,0x69,0x64,0x65,0x73,0x0a,0x70,0x75,0x74,0x0a,0x71,0x0a,0x71,0x75,0x65,0x0a, + 0x71,0x75,0x69,0x63,0x6b,0x6c,0x79,0x0a,0x71,0x75,0x69,0x74,0x65,0x0a,0x71,0x76, + 0x0a,0x72,0x0a,0x72,0x61,0x6e,0x0a,0x72,0x61,0x74,0x68,0x65,0x72,0x0a,0x72,0x64, + 0x0a,0x72,0x65,0x0a,0x72,0x65,0x61,0x64,0x69,0x6c,0x79,0x0a,0x72,0x65,0x61,0x6c, + 0x6c,0x79,0x0a,0x72,0x65,0x63,0x65,0x6e,0x74,0x0a,0x72,0x65,0x63,0x65,0x6e,0x74, + 0x6c,0x79,0x0a,0x72,0x65,0x66,0x0a,0x72,0x65,0x66,0x73,0x0a,0x72,0x65,0x67,0x61, + 0x72,0x64,0x69,0x6e,0x67,0x0a,0x72,0x65,0x67,0x61,0x72,0x64,0x6c,0x65,0x73,0x73, + 0x0a,0x72,0x65,0x67,0x61,0x72,0x64,0x73,0x0a,0x72,0x65,0x6c,0x61,0x74,0x65,0x64, + 0x0a,0x72,0x65,0x6c,0x61,0x74,0x69,0x76,0x65,0x6c,0x79,0x0a,0x72,0x65,0x73,0x65, + 0x61,0x72,0x63,0x68,0x0a,0x72,0x65,0x73,0x70,0x65,0x63,0x74,0x69,0x76,0x65,0x6c, + 0x79,0x0a,0x72,0x65,0x73,0x75,0x6c,0x74,0x65,0x64,0x0a,0x72,0x65,0x73,0x75,0x6c, + 0x74,0x69,0x6e,0x67,0x0a,0x72,0x65,0x73,0x75,0x6c,0x74,0x73,0x0a,0x72,0x69,0x67, + 0x68,0x74,0x0a,0x72,0x75,0x6e,0x0a,0x73,0x0a,0x73,0x61,0x69,0x64,0x0a,0x73,0x61, + 0x6d,0x65,0x0a,0x73,0x61,0x77,0x0a,0x73,0x61,0x79,0x0a,0x73,0x61,0x79,0x69,0x6e, + 0x67,0x0a,0x73,0x61,0x79,0x73,0x0a,0x73,0x65,0x63,0x0a,0x73,0x65,0x63,0x74,0x69, + 0x6f,0x6e,0x0a,0x73,0x65,0x65,0x0a,0x73,0x65,0x65,0x69,0x6e,0x67,0x0a,0x73,0x65, + 0x65,0x6d,0x0a,0x73,0x65,0x65,0x6d,0x65,0x64,0x0a,0x73,0x65,0x65,0x6d,0x69,0x6e, + 0x67,0x0a,0x73,0x65,0x65,0x6d,0x73,0x0a,0x73,0x65,0x65,0x6e,0x0a,0x73,0x65,0x6c, + 0x66,0x0a,0x73,0x65,0x6c,0x76,0x65,0x73,0x0a,0x73,0x65,0x6e,0x74,0x0a,0x73,0x65, + 0x76,0x65,0x6e,0x0a,0x73,0x65,0x76,0x65,0x72,0x61,0x6c,0x0a,0x73,0x68,0x61,0x6c, + 0x6c,0x0a,0x73,0x68,0x65,0x0a,0x73,0x68,0x65,0x64,0x0a,0x73,0x68,0x65,0x27,0x6c, + 0x6c,0x0a,0x73,0x68,0x65,0x73,0x0a,0x73,0x68,0x6f,0x75,0x6c,0x64,0x0a,0x73,0x68, + 0x6f,0x75,0x6c,0x64,0x6e,0x27,0x74,0x0a,0x73,0x68,0x6f,0x77,0x0a,0x73,0x68,0x6f, + 0x77,0x65,0x64,0x0a,0x73,0x68,0x6f,0x77,0x6e,0x0a,0x73,0x68,0x6f,0x77,0x6e,0x73, + 0x0a,0x73,0x68,0x6f,0x77,0x73,0x0a,0x73,0x69,0x67,0x6e,0x69,0x66,0x69,0x63,0x61, + 0x6e,0x74,0x0a,0x73,0x69,0x67,0x6e,0x69,0x66,0x69,0x63,0x61,0x6e,0x74,0x6c,0x79, + 0x0a,0x73,0x69,0x6d,0x69,0x6c,0x61,0x72,0x0a,0x73,0x69,0x6d,0x69,0x6c,0x61,0x72, + 0x6c,0x79,0x0a,0x73,0x69,0x6e,0x63,0x65,0x0a,0x73,0x69,0x78,0x0a,0x73,0x6c,0x69, + 0x67,0x68,0x74,0x6c,0x79,0x0a,0x73,0x6f,0x0a,0x73,0x6f,0x6d,0x65,0x0a,0x73,0x6f, + 0x6d,0x65,0x62,0x6f,0x64,0x79,0x0a,0x73,0x6f,0x6d,0x65,0x68,0x6f,0x77,0x0a,0x73, + 0x6f,0x6d,0x65,0x6f,0x6e,0x65,0x0a,0x73,0x6f,0x6d,0x65,0x74,0x68,0x61,0x6e,0x0a, + 0x73,0x6f,0x6d,0x65,0x74,0x68,0x69,0x6e,0x67,0x0a,0x73,0x6f,0x6d,0x65,0x74,0x69, + 0x6d,0x65,0x0a,0x73,0x6f,0x6d,0x65,0x74,0x69,0x6d,0x65,0x73,0x0a,0x73,0x6f,0x6d, + 0x65,0x77,0x68,0x61,0x74,0x0a,0x73,0x6f,0x6d,0x65,0x77,0x68,0x65,0x72,0x65,0x0a, + 0x73,0x6f,0x6f,0x6e,0x0a,0x73,0x6f,0x72,0x72,0x79,0x0a,0x73,0x70,0x65,0x63,0x69, + 0x66,0x69,0x63,0x61,0x6c,0x6c,0x79,0x0a,0x73,0x70,0x65,0x63,0x69,0x66,0x69,0x65, + 0x64,0x0a,0x73,0x70,0x65,0x63,0x69,0x66,0x79,0x0a,0x73,0x70,0x65,0x63,0x69,0x66, + 0x79,0x69,0x6e,0x67,0x0a,0x73,0x74,0x61,0x74,0x65,0x0a,0x73,0x74,0x61,0x74,0x65, + 0x73,0x0a,0x73,0x74,0x69,0x6c,0x6c,0x0a,0x73,0x74,0x6f,0x70,0x0a,0x73,0x74,0x72, + 0x6f,0x6e,0x67,0x6c,0x79,0x0a,0x73,0x75,0x62,0x0a,0x73,0x75,0x62,0x73,0x74,0x61, + 0x6e,0x74,0x69,0x61,0x6c,0x6c,0x79,0x0a,0x73,0x75,0x63,0x63,0x65,0x73,0x73,0x66, + 0x75,0x6c,0x6c,0x79,0x0a,0x73,0x75,0x63,0x68,0x0a,0x73,0x75,0x66,0x66,0x69,0x63, + 0x69,0x65,0x6e,0x74,0x6c,0x79,0x0a,0x73,0x75,0x67,0x67,0x65,0x73,0x74,0x0a,0x73, + 0x75,0x70,0x0a,0x73,0x75,0x72,0x65,0x0a,0x09,0x74,0x0a,0x74,0x61,0x6b,0x65,0x0a, + 0x74,0x61,0x6b,0x65,0x6e,0x0a,0x74,0x61,0x6b,0x69,0x6e,0x67,0x0a,0x74,0x65,0x6c, + 0x6c,0x0a,0x74,0x65,0x6e,0x64,0x73,0x0a,0x74,0x68,0x0a,0x74,0x68,0x61,0x6e,0x0a, + 0x74,0x68,0x61,0x6e,0x6b,0x0a,0x74,0x68,0x61,0x6e,0x6b,0x73,0x0a,0x74,0x68,0x61, + 0x6e,0x78,0x0a,0x74,0x68,0x61,0x74,0x0a,0x74,0x68,0x61,0x74,0x27,0x6c,0x6c,0x0a, + 0x74,0x68,0x61,0x74,0x73,0x0a,0x74,0x68,0x61,0x74,0x27,0x76,0x65,0x0a,0x74,0x68, + 0x65,0x0a,0x74,0x68,0x65,0x69,0x72,0x0a,0x74,0x68,0x65,0x69,0x72,0x73,0x0a,0x74, + 0x68,0x65,0x6d,0x0a,0x74,0x68,0x65,0x6d,0x73,0x65,0x6c,0x76,0x65,0x73,0x0a,0x74, + 0x68,0x65,0x6e,0x0a,0x74,0x68,0x65,0x6e,0x63,0x65,0x0a,0x74,0x68,0x65,0x72,0x65, + 0x0a,0x74,0x68,0x65,0x72,0x65,0x61,0x66,0x74,0x65,0x72,0x0a,0x74,0x68,0x65,0x72, + 0x65,0x62,0x79,0x0a,0x74,0x68,0x65,0x72,0x65,0x64,0x0a,0x74,0x68,0x65,0x72,0x65, + 0x66,0x6f,0x72,0x65,0x0a,0x74,0x68,0x65,0x72,0x65,0x69,0x6e,0x0a,0x74,0x68,0x65, + 0x72,0x65,0x27,0x6c,0x6c,0x0a,0x74,0x68,0x65,0x72,0x65,0x6f,0x66,0x0a,0x74,0x68, + 0x65,0x72,0x65,0x72,0x65,0x0a,0x74,0x68,0x65,0x72,0x65,0x73,0x0a,0x74,0x68,0x65, + 0x72,0x65,0x74,0x6f,0x0a,0x74,0x68,0x65,0x72,0x65,0x75,0x70,0x6f,0x6e,0x0a,0x74, + 0x68,0x65,0x72,0x65,0x27,0x76,0x65,0x0a,0x74,0x68,0x65,0x73,0x65,0x0a,0x74,0x68, + 0x65,0x79,0x0a,0x74,0x68,0x65,0x79,0x64,0x0a,0x74,0x68,0x65,0x79,0x27,0x6c,0x6c, + 0x0a,0x74,0x68,0x65,0x79,0x72,0x65,0x0a,0x74,0x68,0x65,0x79,0x27,0x76,0x65,0x0a, + 0x74,0x68,0x69,0x6e,0x6b,0x0a,0x74,0x68,0x69,0x73,0x0a,0x74,0x68,0x6f,0x73,0x65, + 0x0a,0x74,0x68,0x6f,0x75,0x0a,0x74,0x68,0x6f,0x75,0x67,0x68,0x0a,0x74,0x68,0x6f, + 0x75,0x67,0x68,0x68,0x0a,0x74,0x68,0x6f,0x75,0x73,0x61,0x6e,0x64,0x0a,0x74,0x68, + 0x72,0x6f,0x75,0x67,0x0a,0x74,0x68,0x72,0x6f,0x75,0x67,0x68,0x0a,0x74,0x68,0x72, + 0x6f,0x75,0x67,0x68,0x6f,0x75,0x74,0x0a,0x74,0x68,0x72,0x75,0x0a,0x74,0x68,0x75, + 0x73,0x0a,0x74,0x69,0x6c,0x0a,0x74,0x69,0x70,0x0a,0x74,0x6f,0x0a,0x74,0x6f,0x67, + 0x65,0x74,0x68,0x65,0x72,0x0a,0x74,0x6f,0x6f,0x0a,0x74,0x6f,0x6f,0x6b,0x0a,0x74, + 0x6f,0x77,0x61,0x72,0x64,0x0a,0x74,0x6f,0x77,0x61,0x72,0x64,0x73,0x0a,0x74,0x72, + 0x69,0x65,0x64,0x0a,0x74,0x72,0x69,0x65,0x73,0x0a,0x74,0x72,0x75,0x6c,0x79,0x0a, + 0x74,0x72,0x79,0x0a,0x74,0x72,0x79,0x69,0x6e,0x67,0x0a,0x74,0x73,0x0a,0x74,0x77, + 0x69,0x63,0x65,0x0a,0x74,0x77,0x6f,0x0a,0x75,0x0a,0x75,0x6e,0x0a,0x75,0x6e,0x64, + 0x65,0x72,0x0a,0x75,0x6e,0x66,0x6f,0x72,0x74,0x75,0x6e,0x61,0x74,0x65,0x6c,0x79, + 0x0a,0x75,0x6e,0x6c,0x65,0x73,0x73,0x0a,0x75,0x6e,0x6c,0x69,0x6b,0x65,0x0a,0x75, + 0x6e,0x6c,0x69,0x6b,0x65,0x6c,0x79,0x0a,0x75,0x6e,0x74,0x69,0x6c,0x0a,0x75,0x6e, + 0x74,0x6f,0x0a,0x75,0x70,0x0a,0x75,0x70,0x6f,0x6e,0x0a,0x75,0x70,0x73,0x0a,0x75, + 0x73,0x0a,0x75,0x73,0x65,0x0a,0x75,0x73,0x65,0x64,0x0a,0x75,0x73,0x65,0x66,0x75, + 0x6c,0x0a,0x75,0x73,0x65,0x66,0x75,0x6c,0x6c,0x79,0x0a,0x75,0x73,0x65,0x66,0x75, + 0x6c,0x6e,0x65,0x73,0x73,0x0a,0x75,0x73,0x65,0x73,0x0a,0x75,0x73,0x69,0x6e,0x67, + 0x0a,0x75,0x73,0x75,0x61,0x6c,0x6c,0x79,0x0a,0x76,0x0a,0x76,0x61,0x6c,0x75,0x65, + 0x0a,0x76,0x61,0x72,0x69,0x6f,0x75,0x73,0x0a,0x27,0x76,0x65,0x0a,0x76,0x65,0x72, + 0x79,0x0a,0x76,0x69,0x61,0x0a,0x76,0x69,0x7a,0x0a,0x76,0x6f,0x6c,0x0a,0x76,0x6f, + 0x6c,0x73,0x0a,0x76,0x73,0x0a,0x77,0x0a,0x77,0x61,0x6e,0x74,0x0a,0x77,0x61,0x6e, + 0x74,0x73,0x0a,0x77,0x61,0x73,0x0a,0x77,0x61,0x73,0x6e,0x27,0x74,0x0a,0x77,0x61, + 0x79,0x0a,0x77,0x65,0x0a,0x77,0x65,0x64,0x0a,0x77,0x65,0x6c,0x63,0x6f,0x6d,0x65, + 0x0a,0x77,0x65,0x27,0x6c,0x6c,0x0a,0x77,0x65,0x6e,0x74,0x0a,0x77,0x65,0x72,0x65, + 0x0a,0x77,0x65,0x72,0x65,0x6e,0x27,0x74,0x0a,0x77,0x65,0x27,0x76,0x65,0x0a,0x77, + 0x68,0x61,0x74,0x0a,0x77,0x68,0x61,0x74,0x65,0x76,0x65,0x72,0x0a,0x77,0x68,0x61, + 0x74,0x27,0x6c,0x6c,0x0a,0x77,0x68,0x61,0x74,0x73,0x0a,0x77,0x68,0x65,0x6e,0x0a, + 0x77,0x68,0x65,0x6e,0x63,0x65,0x0a,0x77,0x68,0x65,0x6e,0x65,0x76,0x65,0x72,0x0a, + 0x77,0x68,0x65,0x72,0x65,0x0a,0x77,0x68,0x65,0x72,0x65,0x61,0x66,0x74,0x65,0x72, + 0x0a,0x77,0x68,0x65,0x72,0x65,0x61,0x73,0x0a,0x77,0x68,0x65,0x72,0x65,0x62,0x79, + 0x0a,0x77,0x68,0x65,0x72,0x65,0x69,0x6e,0x0a,0x77,0x68,0x65,0x72,0x65,0x73,0x0a, + 0x77,0x68,0x65,0x72,0x65,0x75,0x70,0x6f,0x6e,0x0a,0x77,0x68,0x65,0x72,0x65,0x76, + 0x65,0x72,0x0a,0x77,0x68,0x65,0x74,0x68,0x65,0x72,0x0a,0x77,0x68,0x69,0x63,0x68, + 0x0a,0x77,0x68,0x69,0x6c,0x65,0x0a,0x77,0x68,0x69,0x6d,0x0a,0x77,0x68,0x69,0x74, + 0x68,0x65,0x72,0x0a,0x77,0x68,0x6f,0x0a,0x77,0x68,0x6f,0x64,0x0a,0x77,0x68,0x6f, + 0x65,0x76,0x65,0x72,0x0a,0x77,0x68,0x6f,0x6c,0x65,0x0a,0x77,0x68,0x6f,0x27,0x6c, + 0x6c,0x0a,0x77,0x68,0x6f,0x6d,0x0a,0x77,0x68,0x6f,0x6d,0x65,0x76,0x65,0x72,0x0a, + 0x77,0x68,0x6f,0x73,0x0a,0x77,0x68,0x6f,0x73,0x65,0x0a,0x77,0x68,0x79,0x0a,0x77, + 0x69,0x64,0x65,0x6c,0x79,0x0a,0x77,0x69,0x6c,0x6c,0x69,0x6e,0x67,0x0a,0x77,0x69, + 0x73,0x68,0x0a,0x77,0x69,0x74,0x68,0x0a,0x77,0x69,0x74,0x68,0x69,0x6e,0x0a,0x77, + 0x69,0x74,0x68,0x6f,0x75,0x74,0x0a,0x77,0x6f,0x6e,0x27,0x74,0x0a,0x77,0x6f,0x72, + 0x64,0x73,0x0a,0x77,0x6f,0x72,0x6c,0x64,0x0a,0x77,0x6f,0x75,0x6c,0x64,0x0a,0x77, + 0x6f,0x75,0x6c,0x64,0x6e,0x27,0x74,0x0a,0x77,0x77,0x77,0x0a,0x78,0x0a,0x79,0x0a, + 0x79,0x65,0x73,0x0a,0x79,0x65,0x74,0x0a,0x79,0x6f,0x75,0x0a,0x79,0x6f,0x75,0x64, + 0x0a,0x79,0x6f,0x75,0x27,0x6c,0x6c,0x0a,0x79,0x6f,0x75,0x72,0x0a,0x79,0x6f,0x75, + 0x72,0x65,0x0a,0x79,0x6f,0x75,0x72,0x73,0x0a,0x79,0x6f,0x75,0x72,0x73,0x65,0x6c, + 0x66,0x0a,0x79,0x6f,0x75,0x72,0x73,0x65,0x6c,0x76,0x65,0x73,0x0a,0x79,0x6f,0x75, + 0x27,0x76,0x65,0x0a,0x7a,0x0a,0x7a,0x65,0x72,0x6f,0x0a + }; + +static const unsigned char stopwords_fra[]={ + 0x61,0x6c,0x6f,0x72,0x73,0x0a,0x61,0x75,0x0a,0x61,0x75,0x63,0x75,0x6e,0x73,0x0a, + 0x61,0x75,0x73,0x73,0x69,0x0a,0x61,0x75,0x74,0x72,0x65,0x0a,0x61,0x76,0x61,0x6e, + 0x74,0x0a,0x61,0x76,0x65,0x63,0x0a,0x61,0x76,0x6f,0x69,0x72,0x0a,0x62,0x6f,0x6e, + 0x0a,0x63,0x61,0x72,0x0a,0x63,0x65,0x0a,0x63,0x65,0x6c,0x61,0x0a,0x63,0x65,0x73, + 0x0a,0x63,0x65,0x75,0x78,0x0a,0x63,0x68,0x61,0x71,0x75,0x65,0x0a,0x63,0x69,0x0a, + 0x63,0x6f,0x6d,0x6d,0x65,0x0a,0x63,0x6f,0x6d,0x6d,0x65,0x6e,0x74,0x0a,0x64,0x61, + 0x6e,0x73,0x0a,0x64,0x65,0x73,0x0a,0x64,0x75,0x0a,0x64,0x65,0x64,0x61,0x6e,0x73, + 0x0a,0x64,0x65,0x68,0x6f,0x72,0x73,0x0a,0x64,0x65,0x70,0x75,0x69,0x73,0x0a,0x64, + 0x65,0x75,0x78,0x0a,0x64,0x65,0x76,0x72,0x61,0x69,0x74,0x0a,0x64,0x6f,0x69,0x74, + 0x0a,0x64,0x6f,0x6e,0x63,0x0a,0x64,0x6f,0x73,0x0a,0x64,0x72,0x6f,0x69,0x74,0x65, + 0x0a,0x64,0xc3,0xa9,0x62,0x75,0x74,0x0a,0x65,0x6c,0x6c,0x65,0x0a,0x65,0x6c,0x6c, + 0x65,0x73,0x0a,0x65,0x6e,0x0a,0x65,0x6e,0x63,0x6f,0x72,0x65,0x0a,0x65,0x73,0x73, + 0x61,0x69,0x0a,0x65,0x73,0x74,0x0a,0x65,0x74,0x0a,0x65,0x75,0x0a,0x66,0x61,0x69, + 0x74,0x0a,0x66,0x61,0x69,0x74,0x65,0x73,0x0a,0x66,0x6f,0x69,0x73,0x0a,0x66,0x6f, + 0x6e,0x74,0x0a,0x66,0x6f,0x72,0x63,0x65,0x0a,0x68,0x61,0x75,0x74,0x0a,0x68,0x6f, + 0x72,0x73,0x0a,0x69,0x63,0x69,0x0a,0x69,0x6c,0x0a,0x69,0x6c,0x73,0x0a,0x6a,0x65, + 0x0a,0x6c,0x61,0x0a,0x6c,0x65,0x0a,0x6c,0x65,0x73,0x0a,0x6c,0x65,0x75,0x72,0x0a, + 0x6c,0xc3,0xa0,0x0a,0x6d,0x61,0x0a,0x6d,0x61,0x69,0x6e,0x74,0x65,0x6e,0x61,0x6e, + 0x74,0x0a,0x6d,0x61,0x69,0x73,0x0a,0x6d,0x65,0x73,0x0a,0x6d,0x69,0x6e,0x65,0x0a, + 0x6d,0x6f,0x69,0x6e,0x73,0x0a,0x6d,0x6f,0x6e,0x0a,0x6d,0x6f,0x74,0x0a,0x6d,0xc3, + 0xaa,0x6d,0x65,0x0a,0x6e,0x69,0x0a,0x6e,0x6f,0x6d,0x6d,0xc3,0xa9,0x73,0x0a,0x6e, + 0x6f,0x74,0x72,0x65,0x0a,0x6e,0x6f,0x75,0x73,0x0a,0x6e,0x6f,0x75,0x76,0x65,0x61, + 0x75,0x78,0x0a,0x6f,0x75,0x0a,0x6f,0xc3,0xb9,0x0a,0x70,0x61,0x72,0x0a,0x70,0x61, + 0x72,0x63,0x65,0x0a,0x70,0x61,0x72,0x6f,0x6c,0x65,0x0a,0x70,0x61,0x73,0x0a,0x70, + 0x65,0x72,0x73,0x6f,0x6e,0x6e,0x65,0x73,0x0a,0x70,0x65,0x75,0x74,0x0a,0x70,0x65, + 0x75,0x0a,0x70,0x69,0xc3,0xa8,0x63,0x65,0x0a,0x70,0x6c,0x75,0x70,0x61,0x72,0x74, + 0x0a,0x70,0x6f,0x75,0x72,0x0a,0x70,0x6f,0x75,0x72,0x71,0x75,0x6f,0x69,0x0a,0x71, + 0x75,0x61,0x6e,0x64,0x0a,0x71,0x75,0x65,0x0a,0x71,0x75,0x65,0x6c,0x0a,0x71,0x75, + 0x65,0x6c,0x6c,0x65,0x0a,0x71,0x75,0x65,0x6c,0x6c,0x65,0x73,0x0a,0x71,0x75,0x65, + 0x6c,0x73,0x0a,0x71,0x75,0x69,0x0a,0x73,0x61,0x0a,0x73,0x61,0x6e,0x73,0x0a,0x73, + 0x65,0x73,0x0a,0x73,0x65,0x75,0x6c,0x65,0x6d,0x65,0x6e,0x74,0x0a,0x73,0x69,0x0a, + 0x73,0x69,0x65,0x6e,0x0a,0x73,0x6f,0x6e,0x0a,0x73,0x6f,0x6e,0x74,0x0a,0x73,0x6f, + 0x75,0x73,0x0a,0x73,0x6f,0x79,0x65,0x7a,0x0a,0x73,0x75,0x72,0x0a,0x74,0x61,0x0a, + 0x74,0x61,0x6e,0x64,0x69,0x73,0x0a,0x74,0x65,0x6c,0x6c,0x65,0x6d,0x65,0x6e,0x74, + 0x0a,0x74,0x65,0x6c,0x73,0x0a,0x74,0x65,0x73,0x0a,0x74,0x6f,0x6e,0x0a,0x74,0x6f, + 0x75,0x73,0x0a,0x74,0x6f,0x75,0x74,0x0a,0x74,0x72,0x6f,0x70,0x0a,0x74,0x72,0xc3, + 0xa8,0x73,0x0a,0x74,0x75,0x0a,0x76,0x61,0x6c,0x65,0x75,0x72,0x0a,0x76,0x6f,0x69, + 0x65,0x0a,0x76,0x6f,0x69,0x65,0x6e,0x74,0x0a,0x76,0x6f,0x6e,0x74,0x0a,0x76,0x6f, + 0x74,0x72,0x65,0x0a,0x76,0x6f,0x75,0x73,0x0a,0x76,0x75,0x0a,0xc3,0xa7,0x61,0x0a, + 0xc3,0xa9,0x74,0x61,0x69,0x65,0x6e,0x74,0x0a,0xc3,0xa9,0x74,0x61,0x74,0x0a,0xc3, + 0xa9,0x74,0x69,0x6f,0x6e,0x73,0x0a,0xc3,0xa9,0x74,0xc3,0xa9,0x0a,0xc3,0xaa,0x74, + 0x72,0x65,0x0a + }; + +static const unsigned char stopwords_he[]={ + 0xd7,0xa9,0xd7,0x9c,0x0d,0x0a,0xd7,0x90,0xd7,0xaa,0x0d,0x0a,0xd7,0xa2,0xd7,0x9c, + 0x0d,0x0a,0xd7,0x9c,0xd7,0x90,0x0d,0x0a,0xd7,0x9b,0xd7,0x99,0x0d,0x0a,0xd7,0xa2, + 0xd7,0x9d,0x0d,0x0a,0xd7,0x94,0xd7,0x95,0xd7,0x90,0x0d,0x0a,0xd7,0x92,0xd7,0x9d, + 0x0d,0x0a,0xd7,0x91,0x0d,0x0a,0xd7,0x96,0xd7,0x94,0x0d,0x0a,0xd7,0x94,0xd7,0x99, + 0xd7,0x90,0x0d,0x0a,0xd7,0x9b,0xd7,0x9c,0x0d,0x0a,0xd7,0x99,0xd7,0x95,0xd7,0xaa, + 0xd7,0xa8,0x0d,0x0a,0xd7,0x90,0xd7,0x95,0x0d,0x0a,0xd7,0x90,0xd7,0x91,0xd7,0x9c, + 0x0d,0x0a,0xd7,0x91,0xd7,0x99,0xd7,0x9f,0x0d,0x0a,0xd7,0x94,0xd7,0x99,0xd7,0x94, + 0x0d,0x0a,0xd7,0x90,0xd7,0x9d,0x0d,0x0a,0xd7,0x9e,0xd7,0x99,0xd7,0x9c,0xd7,0x99, + 0xd7,0x95,0xd7,0x9f,0x0d,0x0a,0xd7,0x99,0xd7,0xa9,0x0d,0x0a,0xd7,0x9b,0xd7,0x9a, + 0x0d,0x0a,0xd7,0x90,0xd7,0xa0,0xd7,0x99,0x0d,0x0a,0xd7,0x94,0xd7,0x9d,0x0d,0x0a, + 0xd7,0x93,0xd7,0x95,0xd7,0x9c,0xd7,0xa8,0x0d,0x0a,0xd7,0x90,0xd7,0x9e,0xd7,0xa8, + 0x0d,0x0a,0xd7,0xa2,0xd7,0x93,0x0d,0x0a,0xd7,0x9c,0xd7,0x90,0xd7,0x97,0xd7,0xa8, + 0x0d,0x0a,0xd7,0x99,0xd7,0xa9,0xd7,0xa8,0xd7,0x90,0xd7,0x9c,0x0d,0x0a,0xd7,0xa8, + 0xd7,0xa7,0x0d,0x0a,0xd7,0xa9,0xd7,0xa7,0xd7,0x9c,0x0d,0x0a,0xd7,0x9b,0xd7,0x93, + 0xd7,0x99,0x0d,0x0a,0xd7,0x9e,0xd7,0x94,0x0d,0x0a,0xd7,0x9c,0xd7,0xa4,0xd7,0xa0, + 0xd7,0x99,0x0d,0x0a,0xd7,0x90,0xd7,0x97,0xd7,0x93,0x0d,0x0a,0xd7,0x94,0xd7,0x97, + 0xd7,0x91,0xd7,0xa8,0xd7,0x94,0x0d,0x0a,0xd7,0x9b,0xd7,0x9e,0xd7,0x95,0x0d,0x0a, + 0xd7,0x96,0xd7,0x90,0xd7,0xaa,0x0d,0x0a,0xd7,0x94,0xd7,0x99,0xd7,0x95,0xd7,0x9d, + 0x0d,0x0a,0xd7,0x90,0xd7,0x9a,0x0d,0x0a,0xd7,0x9c,0x0d,0x0a,0xd7,0x94,0x0d,0x0a, + 0xd7,0x9b,0x0d,0x0a,0xd7,0x90,0xd7,0x99,0xd7,0x9f,0x0d,0x0a,0xd7,0x90,0xd7,0xaa, + 0xd7,0x9e,0xd7,0x95,0xd7,0x9c,0x0d,0x0a,0xd7,0xa9,0xd7,0x9c,0xd7,0x90,0x0d,0x0a, + 0xd7,0x9b,0xd7,0x91,0xd7,0xa8,0x0d,0x0a,0xd7,0xa2,0xd7,0x95,0xd7,0x93,0x0d,0x0a, + 0xd7,0x9c,0xd7,0x95,0x0d,0x0a,0xd7,0x96,0xd7,0x95,0x0d,0x0a,0xd7,0x90,0xd7,0x9c, + 0x0d,0x0a,0xd7,0x91,0xd7,0x9f,0x0d,0x0a,0xd7,0x90,0xd7,0x95,0xd7,0xaa,0xd7,0x95, + 0x0d,0x0a,0xd7,0xa9,0xd7,0xa0,0xd7,0x99,0x0d,0x0a,0xd7,0x91,0xd7,0x99,0xd7,0xaa, + 0x0d,0x0a,0xd7,0x99,0xd7,0x93,0xd7,0x99,0x0d,0x0a,0xd7,0x9b,0xd7,0x9e,0xd7,0x94, + 0x0d,0x0a,0xd7,0x91,0xd7,0x99,0xd7,0x95,0xd7,0xaa,0xd7,0xa8,0x0d,0x0a,0xd7,0x95, + 0xd7,0x9c,0xd7,0x90,0x0d,0x0a,0xd7,0x94,0xd7,0x9e,0xd7,0x9e,0xd7,0xa9,0xd7,0x9c, + 0xd7,0x94,0x0d,0x0a,0xd7,0x90,0xd7,0x97,0xd7,0xa8,0xd7,0x99,0x0d,0x0a,0xd7,0x97, + 0xd7,0x91,0xd7,0xa8,0xd7,0xaa,0x0d,0x0a,0xd7,0x94,0xd7,0x99,0xd7,0xaa,0xd7,0x94, + 0x0d,0x0a,0xd7,0xa9,0xd7,0x9c,0xd7,0x95,0x0d,0x0a,0xd7,0x94,0xd7,0x99,0xd7,0x95, + 0x0d,0x0a,0xd7,0xa0,0xd7,0x92,0xd7,0x93,0x0d,0x0a,0xd7,0x91,0xd7,0x9b,0xd7,0x9c, + 0x0d,0x0a,0xd7,0x90,0xd7,0x91,0xd7,0x99,0xd7,0x91,0x0d,0x0a,0xd7,0xa8,0xd7,0x90, + 0xd7,0xa9,0x0d,0x0a,0xd7,0x91,0xd7,0x99,0xd7,0xa9,0xd7,0xa8,0xd7,0x90,0xd7,0x9c, + 0x0d,0x0a,0xd7,0x9c,0xd7,0x99,0x0d,0x0a,0xd7,0xa9,0xd7,0xa0,0xd7,0x99,0xd7,0x9d, + 0x0d,0x0a,0xd7,0xa4,0xd7,0x99,0x0d,0x0a,0xd7,0x91,0xd7,0x95,0x0d,0x0a,0xd7,0x9e, + 0x0d,0x0a,0xd7,0x9e,0xd7,0x90,0xd7,0x95,0xd7,0x93,0x0d,0x0a,0xd7,0x9c,0xd7,0x94, + 0xd7,0x99,0xd7,0x95,0xd7,0xaa,0x0d,0x0a,0xd7,0xa9,0xd7,0x94,0xd7,0x95,0xd7,0x90, + 0x0d,0x0a,0xd7,0x9e,0xd7,0x99,0x0d,0x0a,0xd7,0x90,0xd7,0x9c,0xd7,0xa3,0x0d,0x0a, + 0xd7,0x90,0xd7,0x9c,0xd7,0x90,0x0d,0x0a,0xd7,0x90,0xd7,0xa3,0x0d,0x0a,0xd7,0x90, + 0xd7,0x97,0xd7,0xa8,0x0d,0x0a,0xd7,0x94,0xd7,0x96,0xd7,0x94,0x0d,0x0a,0xd7,0x90, + 0xd7,0x97,0xd7,0xaa,0x0d,0x0a,0xd7,0x91,0xd7,0x91,0xd7,0x99,0xd7,0xaa,0x0d,0x0a, + 0xd7,0x90,0xd7,0x9c,0xd7,0x94,0x0d,0x0a,0xd7,0x90,0xd7,0xa0,0xd7,0x97,0xd7,0xa0, + 0xd7,0x95,0x0d,0x0a + }; + +static std::map<std::string, std::pair<const unsigned char*, unsigned int> > createResourceMap() { + std::map<std::string, std::pair<const unsigned char*, unsigned int> > m; + m["stopwords/en"] = std::pair <const unsigned char*, unsigned int>(stopwords_en, sizeof stopwords_en); + m["stopwords/fra"] = std::pair <const unsigned char*, unsigned int>(stopwords_fra, sizeof stopwords_fra); + m["stopwords/he"] = std::pair <const unsigned char*, unsigned int>(stopwords_he, sizeof stopwords_he); + return m; +} + +static std::map<std::string, std::pair<const unsigned char*, unsigned int> > resourceMap = createResourceMap(); + +std::string getResourceAsString(const std::string &name) { + std::map<std::string, std::pair<const unsigned char*, unsigned int> >::iterator it = resourceMap.find(name); + if (it != resourceMap.end()) { + return std::string((const char*)resourceMap[name].first, resourceMap[name].second); + } + return ""; +} diff --git a/zimwriterfs/resourceTools.h b/zimwriterfs/resourceTools.h new file mode 100644 index 0000000..c040f86 --- /dev/null +++ b/zimwriterfs/resourceTools.h @@ -0,0 +1,10 @@ +#ifndef OPENZIM_ZIMWRITERFS_RESOURCETOOLS_H +#define OPENZIM_ZIMWRITERFS_RESOURCETOOLS_H + +#include <string> + +std::string getResourceAsString(const std::string &name); + + +#endif // OPENZIM_ZIMWRITERFS_RESOURCETOOLS_H + diff --git a/zimwriterfs/zimwriterfs.cpp b/zimwriterfs/zimwriterfs.cpp index 1826c31..52ed7ea 100644 --- a/zimwriterfs/zimwriterfs.cpp +++ b/zimwriterfs/zimwriterfs.cpp @@ -58,6 +58,7 @@ pthread_mutex_t verboseMutex; bool inflateHtmlFlag = false; bool uniqueNamespace = false; +bool createFullTextIndex = false; magic_t magic; @@ -132,6 +133,7 @@ std::cout << "\t-x, --inflateHtml\ttry to inflate HTML files before packing (*.html, *.htm, ...)" << std::endl; std::cout << "\t-u, --uniqueNamespace\tput everything in the same namespace 'A'. Might be necessary to avoid problems with dynamic/javascript data loading." << std::endl; std::cout << "\t-r, --redirects\t\tpath to the CSV file with the list of redirects (url, title, target_url tab separated)." << std::endl; + std::cout << "\t-i, --createFullTextIndex\t\tIndex the content and add it to the ZIM." << std::endl; std::cout << std::endl; std::cout << "Example:" << std::endl; @@ -228,7 +230,7 @@ int main(int argc, char** argv) { ArticleSource source(filenameQueue); int minChunkSize = 2048; - + /* Argument parsing */ static struct option long_options[] = { @@ -245,13 +247,14 @@ {"description", required_argument, 0, 'd'}, {"creator", required_argument, 0, 'c'}, {"publisher", required_argument, 0, 'p'}, + {"createFullTextIndex", no_argument, 0, 'i'}, {0, 0, 0, 0} }; int option_index = 0; int c; do { - c = getopt_long(argc, argv, "hvxuw:m:f:t:d:c:l:p:r:", long_options, &option_index); + c = getopt_long(argc, argv, "hvixuw:m:f:t:d:c:l:p:r:", long_options, &option_index); if (c != -1) { switch (c) { @@ -274,6 +277,9 @@ case 'f': favicon = optarg; break; + case 'i': + createFullTextIndex = true; + break; case 'l': language = optarg; break; -- To view, visit https://gerrit.wikimedia.org/r/296912 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: merged Gerrit-Change-Id: I55079339d21d6903634c265f83f4d1c6ba0ac333 Gerrit-PatchSet: 1 Gerrit-Project: openzim Gerrit-Branch: master Gerrit-Owner: Mgautierfr <mgaut...@kymeria.fr> Gerrit-Reviewer: Kelson <kel...@kiwix.org> _______________________________________________ MediaWiki-commits mailing list MediaWiki-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits