Kelson has submitted this change and it was merged. (
https://gerrit.wikimedia.org/r/296912 )
Change subject: Add a indexer.
......................................................................
Add a indexer.
This indexer is not used.
This is mainly code from kiwix-indexer imported in openzim.
Unused function in *Tools has been removed.
No dependency to xapian.
Change-Id: I55079339d21d6903634c265f83f4d1c6ba0ac333
---
M zimwriterfs/Makefile.am
A zimwriterfs/indexer.cpp
A zimwriterfs/indexer.h
A zimwriterfs/pathTools.cpp
A zimwriterfs/pathTools.h
A zimwriterfs/resourceTools.cpp
A zimwriterfs/resourceTools.h
M zimwriterfs/zimwriterfs.cpp
8 files changed, 921 insertions(+), 2 deletions(-)
Approvals:
Kelson: Verified; Looks good to me, approved
diff --git a/zimwriterfs/Makefile.am b/zimwriterfs/Makefile.am
index 92641d9..628b74c 100644
--- a/zimwriterfs/Makefile.am
+++ b/zimwriterfs/Makefile.am
@@ -6,4 +6,7 @@
tools.cpp \
article.cpp \
articlesource.cpp \
+ indexer.cpp \
+ resourceTools.cpp \
+ pathTools.cpp \
mimetypecounter.cpp
diff --git a/zimwriterfs/indexer.cpp b/zimwriterfs/indexer.cpp
new file mode 100644
index 0000000..7820a32
--- /dev/null
+++ b/zimwriterfs/indexer.cpp
@@ -0,0 +1,262 @@
+/*
+ * Copyright 2011-2014 Emmanuel Engelhart <[email protected]>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ */
+
+#include "indexer.h"
+#include "resourceTools.h"
+#include "pathTools.h"
+#include <unistd.h>
+
+ /* Count word */
+ unsigned int Indexer::countWords(const string &text) {
+ unsigned int numWords = 1;
+ unsigned int length = text.size();
+
+ for(unsigned int i=0; i<length;) {
+ while(i<length && text[i] != ' ') {
+ i++;
+ }
+ numWords++;
+ i++;
+ }
+
+ return numWords;
+ }
+
+ /* Constructor */
+ Indexer::Indexer() :
+ keywordsBoostFactor(3),
+ verboseFlag(false) {
+
+ /* Initialize mutex */
+ pthread_mutex_init(&threadIdsMutex, NULL);
+ pthread_mutex_init(&toIndexQueueMutex, NULL);
+ pthread_mutex_init(&articleIndexerRunningMutex, NULL);
+ pthread_mutex_init(&articleCountMutex, NULL);
+ pthread_mutex_init(&zimIdMutex, NULL);
+ pthread_mutex_init(&indexPathMutex, NULL);
+ pthread_mutex_init(&verboseMutex, NULL);
+ }
+
+ /* Destructor */
+ Indexer::~Indexer() {
+ }
+
+ /* Read the stopwords */
+ void Indexer::readStopWords(const string languageCode) {
+ std::string stopWord;
+ std::istringstream file(getResourceAsString("stopwords/" + languageCode));
+
+ this->stopWords.clear();
+
+ while (getline(file, stopWord, '\n')) {
+ this->stopWords.push_back(stopWord);
+ }
+ }
+
+ /* Article indexer methods */
+ void *Indexer::indexArticles(void *ptr) {
+ pthread_setcanceltype(PTHREAD_CANCEL_DEFERRED, NULL);
+ Indexer *self = (Indexer *)ptr;
+ unsigned int indexedArticleCount = 0;
+ indexerToken token;
+
+ self->indexingPrelude(self->getIndexPath());
+
+ while (self->popFromToIndexQueue(token)) {
+ self->index(token.url,
+ token.accentedTitle,
+ token.title,
+ token.keywords,
+ token.content,
+ token.snippet,
+ token.size,
+ token.wordCount
+ );
+
+ indexedArticleCount += 1;
+
+ /* Make a hard-disk flush every 10.000 articles */
+ if (indexedArticleCount % 5000 == 0) {
+ self->flush();
+ }
+
+ /* Test if the thread should be cancelled */
+ pthread_testcancel();
+ }
+ self->indexingPostlude();
+
+ /* Write content id file */
+ string path = appendToDirectory(self->getIndexPath(), "content.id");
+ writeTextFile(path, self->getZimId());
+
+ usleep(100);
+
+ self->articleIndexerRunning(false);
+ pthread_exit(NULL);
+ return NULL;
+ }
+
+ void Indexer::articleIndexerRunning(bool value) {
+ pthread_mutex_lock(&articleIndexerRunningMutex);
+ this->articleIndexerRunningFlag = value;
+ pthread_mutex_unlock(&articleIndexerRunningMutex);
+ }
+
+ bool Indexer::isArticleIndexerRunning() {
+ pthread_mutex_lock(&articleIndexerRunningMutex);
+ bool retVal = this->articleIndexerRunningFlag;
+ pthread_mutex_unlock(&articleIndexerRunningMutex);
+ return retVal;
+ }
+
+ /* ToIndexQueue methods */
+ bool Indexer::isToIndexQueueEmpty() {
+ pthread_mutex_lock(&toIndexQueueMutex);
+ bool retVal = this->toIndexQueue.empty();
+ pthread_mutex_unlock(&toIndexQueueMutex);
+ return retVal;
+ }
+
+ void Indexer::pushToIndexQueue(indexerToken &token) {
+ pthread_mutex_lock(&toIndexQueueMutex);
+ this->toIndexQueue.push(token);
+ pthread_mutex_unlock(&toIndexQueueMutex);
+ usleep(int(this->toIndexQueue.size() / 200) / 10 * 1000);
+ }
+
+ bool Indexer::popFromToIndexQueue(indexerToken &token) {
+ while (this->isToIndexQueueEmpty()) {
+ usleep(500);
+ if (this->getVerboseFlag()) {
+ std::cout << "Waiting... ToIndexQueue is empty for now..." << std::endl;
+ }
+
+ pthread_testcancel();
+ }
+
+ pthread_mutex_lock(&toIndexQueueMutex);
+ token = this->toIndexQueue.front();
+ this->toIndexQueue.pop();
+ pthread_mutex_unlock(&toIndexQueueMutex);
+
+ if (token.title == ""){
+ //This is a empty token, end of the queue.
+ return false;
+ }
+ return true;
+ }
+
+ /* Index methods */
+ void Indexer::setIndexPath(const string path) {
+ pthread_mutex_lock(&indexPathMutex);
+ this->indexPath = path;
+ pthread_mutex_unlock(&indexPathMutex);
+ }
+
+ string Indexer::getIndexPath() {
+ pthread_mutex_lock(&indexPathMutex);
+ string retVal = this->indexPath;
+ pthread_mutex_unlock(&indexPathMutex);
+ return retVal;
+ }
+
+ void Indexer::setArticleCount(const unsigned int articleCount) {
+ pthread_mutex_lock(&articleCountMutex);
+ this->articleCount = articleCount;
+ pthread_mutex_unlock(&articleCountMutex);
+ }
+
+ unsigned int Indexer::getArticleCount() {
+ pthread_mutex_lock(&articleCountMutex);
+ unsigned int retVal = this->articleCount;
+ pthread_mutex_unlock(&articleCountMutex);
+ return retVal;
+ }
+
+ void Indexer::setZimId(const string id) {
+ pthread_mutex_lock(&zimIdMutex);
+ this->zimId = id;
+ pthread_mutex_unlock(&zimIdMutex);
+ }
+
+ string Indexer::getZimId() {
+ pthread_mutex_lock(&zimIdMutex);
+ string retVal = this->zimId;
+ pthread_mutex_unlock(&zimIdMutex);
+ return retVal;
+ }
+
+ /* Manage */
+ bool Indexer::start(const string indexPath) {
+ if (this->getVerboseFlag()) {
+ std::cout << "Indexing starting..." <<std::endl;
+ }
+
+ this->setArticleCount(0);
+ this->setIndexPath(indexPath);
+
+ pthread_mutex_lock(&threadIdsMutex);
+
+ this->articleIndexerRunning(true);
+ pthread_create(&(this->articleIndexer), NULL, Indexer::indexArticles,
(void*)this);
+ pthread_detach(this->articleIndexer);
+ pthread_mutex_unlock(&threadIdsMutex);
+
+ return true;
+ }
+
+ bool Indexer::isRunning() {
+ if (this->getVerboseFlag()) {
+ std::cout << "isArticleIndexer running: " <<
(this->isArticleIndexerRunning() ? "yes" : "no") << std::endl;
+ }
+
+ return this->isArticleIndexerRunning();
+ }
+
+ bool Indexer::stop() {
+ if (this->isRunning()) {
+ bool isArticleIndexerRunning = this->isArticleIndexerRunning();
+
+ pthread_mutex_lock(&threadIdsMutex);
+
+ if (isArticleIndexerRunning) {
+ pthread_cancel(this->articleIndexer);
+ this->articleIndexerRunning(false);
+ }
+
+ pthread_mutex_unlock(&threadIdsMutex);
+ }
+
+ return true;
+ }
+
+ /* Manage the verboseFlag */
+ void Indexer::setVerboseFlag(const bool value) {
+ pthread_mutex_lock(&verboseMutex);
+ this->verboseFlag = value;
+ pthread_mutex_unlock(&verboseMutex);
+ }
+
+ bool Indexer::getVerboseFlag() {
+ bool value;
+ pthread_mutex_lock(&verboseMutex);
+ value = this->verboseFlag;
+ pthread_mutex_unlock(&verboseMutex);
+ return value;
+ }
diff --git a/zimwriterfs/indexer.h b/zimwriterfs/indexer.h
new file mode 100644
index 0000000..02d989b
--- /dev/null
+++ b/zimwriterfs/indexer.h
@@ -0,0 +1,134 @@
+/*
+ * Copyright 2014 Emmanuel Engelhart <[email protected]>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ */
+
+#ifndef OPENZIM_ZIMWRITERFS_INDEXER_H
+#define OPENZIM_ZIMWRITERFS_INDEXER_H
+
+#include <string>
+#include <vector>
+#include <stack>
+#include <queue>
+#include <fstream>
+#include <iostream>
+#include <sstream>
+
+#include <pthread.h>
+/*#include <stringTools.h>
+#include <otherTools.h>
+#include <resourceTools.h>*/
+#include <zim/file.h>
+#include <zim/article.h>
+#include <zim/fileiterator.h>
+#include <zim/writer/zimcreator.h>
+/*#include "reader.h"*/
+
+using namespace std;
+
+struct indexerToken {
+ string url;
+ string accentedTitle;
+ string title;
+ string keywords;
+ string content;
+ string snippet;
+ string size;
+ string wordCount;
+};
+
+class Indexer {
+
+ public:
+ Indexer();
+ virtual ~Indexer();
+
+ bool start(const string indexPath);
+ bool stop();
+ bool isRunning();
+ void setVerboseFlag(const bool value);
+ void pushToIndexQueue(indexerToken &token);
+
+ protected:
+ virtual void indexingPrelude(const string indexPath) = 0;
+ virtual void index(const string &url,
+ const string &title,
+ const string &unaccentedTitle,
+ const string &keywords,
+ const string &content,
+ const string &snippet,
+ const string &size,
+ const string &wordCount) = 0;
+ virtual void flush() = 0;
+ virtual void indexingPostlude() = 0;
+
+ /* Stop words */
+ std::vector<std::string> stopWords;
+ void readStopWords(const string languageCode);
+
+ /* Others */
+ unsigned int countWords(const string &text);
+
+ /* Boost factor */
+ unsigned int keywordsBoostFactor;
+ inline unsigned int getTitleBoostFactor(const unsigned int contentLength) {
+ return contentLength / 500 + 1;
+ }
+
+ /* Verbose */
+ pthread_mutex_t verboseMutex;
+ bool getVerboseFlag();
+ bool verboseFlag;
+
+ private:
+ pthread_mutex_t threadIdsMutex;
+
+ /* Index writting */
+ pthread_t articleIndexer;
+ pthread_mutex_t articleIndexerRunningMutex;
+ static void *indexArticles(void *ptr);
+ bool articleIndexerRunningFlag;
+ bool isArticleIndexerRunning();
+ void articleIndexerRunning(bool value);
+
+ /* To index queue */
+ std::queue<indexerToken> toIndexQueue;
+ pthread_mutex_t toIndexQueueMutex;
+ /* void pushToIndexQueue(indexerToken &token); is public */
+ bool popFromToIndexQueue(indexerToken &token);
+ bool isToIndexQueueEmpty();
+
+ /* Article Count & Progression */
+ unsigned int articleCount;
+ pthread_mutex_t articleCountMutex;
+ void setArticleCount(const unsigned int articleCount);
+ unsigned int getArticleCount();
+
+ /* Index path */
+ pthread_mutex_t indexPathMutex;
+ string indexPath;
+ void setIndexPath(const string path);
+ string getIndexPath();
+
+ /* ZIM id */
+ pthread_mutex_t zimIdMutex;
+ string zimId;
+ void setZimId(const string id);
+ string getZimId();
+ };
+
+#endif //OPENZIM_ZIMWRITERFS_INDEXER_H
diff --git a/zimwriterfs/pathTools.cpp b/zimwriterfs/pathTools.cpp
new file mode 100644
index 0000000..3570252
--- /dev/null
+++ b/zimwriterfs/pathTools.cpp
@@ -0,0 +1,78 @@
+/*
+ * Copyright 2011-2014 Emmanuel Engelhart <[email protected]>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ */
+
+#include "pathTools.h"
+
+#ifdef __APPLE__
+#include <mach-o/dyld.h>
+#include <limits.h>
+#elif _WIN32
+#include <windows.h>
+#include "Shlwapi.h"
+#endif
+
+#ifdef _WIN32
+#else
+#include <unistd.h>
+#endif
+
+#ifdef _WIN32
+#define SEPARATOR "\\"
+#else
+#define SEPARATOR "/"
+#endif
+
+#ifndef PATH_MAX
+#define PATH_MAX 1024
+#endif
+
+
+std::string appendToDirectory(const std::string &directoryPath, const
std::string &filename) {
+ std::string newPath = directoryPath + SEPARATOR + filename;
+ return newPath;
+}
+
+
+std::string getExecutablePath() {
+ char binRootPath[PATH_MAX];
+
+#ifdef _WIN32
+ GetModuleFileName( NULL, binRootPath, PATH_MAX);
+ return std::string(binRootPath);
+#elif __APPLE__
+ uint32_t max = (uint32_t)PATH_MAX;
+ _NSGetExecutablePath(binRootPath, &max);
+ return std::string(binRootPath);
+#else
+ ssize_t size = readlink("/proc/self/exe", binRootPath, PATH_MAX);
+ if (size != -1) {
+ return std::string(binRootPath, size);
+ }
+#endif
+
+ return "";
+}
+
+bool writeTextFile(const std::string &path, const std::string &content) {
+ std::ofstream file;
+ file.open(path.c_str());
+ file << content;
+ file.close();
+ return true;
+}
diff --git a/zimwriterfs/pathTools.h b/zimwriterfs/pathTools.h
new file mode 100644
index 0000000..b66486e
--- /dev/null
+++ b/zimwriterfs/pathTools.h
@@ -0,0 +1,46 @@
+/*
+ * Copyright 2011 Emmanuel Engelhart <[email protected]>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ */
+
+#ifndef OPENZIM_ZIMWRITERFS_PATHTOOLS_H
+#define OPENZIM_ZIMWRITERFS_PATHTOOLS_H
+
+#include <string>
+#include <vector>
+#include <sstream>
+#include <iostream>
+#include <fstream>
+#include <string.h>
+#include <stdio.h>
+#include <sys/types.h>
+#include <fcntl.h>
+#include <sys/stat.h>
+#include <ios>
+#include <limits.h>
+
+#ifdef _WIN32
+#include <direct.h>
+#endif
+
+std::string appendToDirectory(const std::string &directoryPath, const
std::string &filename);
+
+std::string getExecutablePath();
+
+bool writeTextFile(const std::string &path, const std::string &content);
+
+#endif //OPENZIM_ZIMWRITERFS_PATHTOOLS_H
diff --git a/zimwriterfs/resourceTools.cpp b/zimwriterfs/resourceTools.cpp
new file mode 100644
index 0000000..c00e5a0
--- /dev/null
+++ b/zimwriterfs/resourceTools.cpp
@@ -0,0 +1,380 @@
+
+#include <resourceTools.h>
+#include <string>
+#include <map>
+
+static const unsigned char stopwords_en[]={
+
0x61,0x0a,0x61,0x62,0x6c,0x65,0x0a,0x61,0x62,0x6f,0x75,0x74,0x0a,0x61,0x62,0x6f,
+
0x76,0x65,0x0a,0x61,0x62,0x73,0x74,0x0a,0x61,0x63,0x63,0x6f,0x72,0x64,0x61,0x6e,
+
0x63,0x65,0x0a,0x61,0x63,0x63,0x6f,0x72,0x64,0x69,0x6e,0x67,0x0a,0x61,0x63,0x63,
+
0x6f,0x72,0x64,0x69,0x6e,0x67,0x6c,0x79,0x0a,0x61,0x63,0x72,0x6f,0x73,0x73,0x0a,
+
0x61,0x63,0x74,0x0a,0x61,0x63,0x74,0x75,0x61,0x6c,0x6c,0x79,0x0a,0x61,0x64,0x64,
+
0x65,0x64,0x0a,0x61,0x64,0x6a,0x0a,0x61,0x64,0x6f,0x70,0x74,0x65,0x64,0x0a,0x61,
+
0x66,0x66,0x65,0x63,0x74,0x65,0x64,0x0a,0x61,0x66,0x66,0x65,0x63,0x74,0x69,0x6e,
+
0x67,0x0a,0x61,0x66,0x66,0x65,0x63,0x74,0x73,0x0a,0x61,0x66,0x74,0x65,0x72,0x0a,
+
0x61,0x66,0x74,0x65,0x72,0x77,0x61,0x72,0x64,0x73,0x0a,0x61,0x67,0x61,0x69,0x6e,
+
0x0a,0x61,0x67,0x61,0x69,0x6e,0x73,0x74,0x0a,0x61,0x68,0x0a,0x61,0x6c,0x6c,0x0a,
+
0x61,0x6c,0x6d,0x6f,0x73,0x74,0x0a,0x61,0x6c,0x6f,0x6e,0x65,0x0a,0x61,0x6c,0x6f,
+
0x6e,0x67,0x0a,0x61,0x6c,0x72,0x65,0x61,0x64,0x79,0x0a,0x61,0x6c,0x73,0x6f,0x0a,
+
0x61,0x6c,0x74,0x68,0x6f,0x75,0x67,0x68,0x0a,0x61,0x6c,0x77,0x61,0x79,0x73,0x0a,
+
0x61,0x6d,0x0a,0x61,0x6d,0x6f,0x6e,0x67,0x0a,0x61,0x6d,0x6f,0x6e,0x67,0x73,0x74,
+
0x0a,0x61,0x6e,0x0a,0x61,0x6e,0x64,0x0a,0x61,0x6e,0x6e,0x6f,0x75,0x6e,0x63,0x65,
+
0x0a,0x61,0x6e,0x6f,0x74,0x68,0x65,0x72,0x0a,0x61,0x6e,0x79,0x0a,0x61,0x6e,0x79,
+
0x62,0x6f,0x64,0x79,0x0a,0x61,0x6e,0x79,0x68,0x6f,0x77,0x0a,0x61,0x6e,0x79,0x6d,
+
0x6f,0x72,0x65,0x0a,0x61,0x6e,0x79,0x6f,0x6e,0x65,0x0a,0x61,0x6e,0x79,0x74,0x68,
+
0x69,0x6e,0x67,0x0a,0x61,0x6e,0x79,0x77,0x61,0x79,0x0a,0x61,0x6e,0x79,0x77,0x61,
+
0x79,0x73,0x0a,0x61,0x6e,0x79,0x77,0x68,0x65,0x72,0x65,0x0a,0x61,0x70,0x70,0x61,
+
0x72,0x65,0x6e,0x74,0x6c,0x79,0x0a,0x61,0x70,0x70,0x72,0x6f,0x78,0x69,0x6d,0x61,
+
0x74,0x65,0x6c,0x79,0x0a,0x61,0x72,0x65,0x0a,0x61,0x72,0x65,0x6e,0x0a,0x61,0x72,
+
0x65,0x6e,0x74,0x0a,0x61,0x72,0x69,0x73,0x65,0x0a,0x61,0x72,0x6f,0x75,0x6e,0x64,
+
0x0a,0x61,0x73,0x0a,0x61,0x73,0x69,0x64,0x65,0x0a,0x61,0x73,0x6b,0x0a,0x61,0x73,
+
0x6b,0x69,0x6e,0x67,0x0a,0x61,0x74,0x0a,0x61,0x75,0x74,0x68,0x0a,0x61,0x76,0x61,
+
0x69,0x6c,0x61,0x62,0x6c,0x65,0x0a,0x61,0x77,0x61,0x79,0x0a,0x61,0x77,0x66,0x75,
+
0x6c,0x6c,0x79,0x0a,0x62,0x0a,0x62,0x61,0x63,0x6b,0x0a,0x62,0x65,0x0a,0x62,0x65,
+
0x63,0x61,0x6d,0x65,0x0a,0x62,0x65,0x63,0x61,0x75,0x73,0x65,0x0a,0x62,0x65,0x63,
+
0x6f,0x6d,0x65,0x0a,0x62,0x65,0x63,0x6f,0x6d,0x65,0x73,0x0a,0x62,0x65,0x63,0x6f,
+
0x6d,0x69,0x6e,0x67,0x0a,0x62,0x65,0x65,0x6e,0x0a,0x62,0x65,0x66,0x6f,0x72,0x65,
+
0x0a,0x62,0x65,0x66,0x6f,0x72,0x65,0x68,0x61,0x6e,0x64,0x0a,0x62,0x65,0x67,0x69,
+
0x6e,0x0a,0x62,0x65,0x67,0x69,0x6e,0x6e,0x69,0x6e,0x67,0x0a,0x62,0x65,0x67,0x69,
+
0x6e,0x6e,0x69,0x6e,0x67,0x73,0x0a,0x62,0x65,0x67,0x69,0x6e,0x73,0x0a,0x62,0x65,
+
0x68,0x69,0x6e,0x64,0x0a,0x62,0x65,0x69,0x6e,0x67,0x0a,0x62,0x65,0x6c,0x69,0x65,
+
0x76,0x65,0x0a,0x62,0x65,0x6c,0x6f,0x77,0x0a,0x62,0x65,0x73,0x69,0x64,0x65,0x0a,
+
0x62,0x65,0x73,0x69,0x64,0x65,0x73,0x0a,0x62,0x65,0x74,0x77,0x65,0x65,0x6e,0x0a,
+
0x62,0x65,0x79,0x6f,0x6e,0x64,0x0a,0x62,0x69,0x6f,0x6c,0x0a,0x62,0x6f,0x74,0x68,
+
0x0a,0x62,0x72,0x69,0x65,0x66,0x0a,0x62,0x72,0x69,0x65,0x66,0x6c,0x79,0x0a,0x62,
+
0x75,0x74,0x0a,0x62,0x79,0x0a,0x63,0x0a,0x63,0x61,0x0a,0x63,0x61,0x6d,0x65,0x0a,
+
0x63,0x61,0x6e,0x0a,0x63,0x61,0x6e,0x6e,0x6f,0x74,0x0a,0x63,0x61,0x6e,0x27,0x74,
+
0x0a,0x63,0x61,0x75,0x73,0x65,0x0a,0x63,0x61,0x75,0x73,0x65,0x73,0x0a,0x63,0x65,
+
0x72,0x74,0x61,0x69,0x6e,0x0a,0x63,0x65,0x72,0x74,0x61,0x69,0x6e,0x6c,0x79,0x0a,
+
0x63,0x6f,0x0a,0x63,0x6f,0x6d,0x0a,0x63,0x6f,0x6d,0x65,0x0a,0x63,0x6f,0x6d,0x65,
+
0x73,0x0a,0x63,0x6f,0x6e,0x74,0x61,0x69,0x6e,0x0a,0x63,0x6f,0x6e,0x74,0x61,0x69,
+
0x6e,0x69,0x6e,0x67,0x0a,0x63,0x6f,0x6e,0x74,0x61,0x69,0x6e,0x73,0x0a,0x63,0x6f,
+
0x75,0x6c,0x64,0x0a,0x63,0x6f,0x75,0x6c,0x64,0x6e,0x74,0x0a,0x64,0x0a,0x64,0x61,
+
0x74,0x65,0x0a,0x64,0x69,0x64,0x0a,0x64,0x69,0x64,0x6e,0x27,0x74,0x0a,0x64,0x69,
+
0x66,0x66,0x65,0x72,0x65,0x6e,0x74,0x0a,0x64,0x6f,0x0a,0x64,0x6f,0x65,0x73,0x0a,
+
0x64,0x6f,0x65,0x73,0x6e,0x27,0x74,0x0a,0x64,0x6f,0x69,0x6e,0x67,0x0a,0x64,0x6f,
+
0x6e,0x65,0x0a,0x64,0x6f,0x6e,0x27,0x74,0x0a,0x64,0x6f,0x77,0x6e,0x0a,0x64,0x6f,
+
0x77,0x6e,0x77,0x61,0x72,0x64,0x73,0x0a,0x64,0x75,0x65,0x0a,0x64,0x75,0x72,0x69,
+
0x6e,0x67,0x0a,0x65,0x0a,0x65,0x61,0x63,0x68,0x0a,0x65,0x64,0x0a,0x65,0x64,0x75,
+
0x0a,0x65,0x66,0x66,0x65,0x63,0x74,0x0a,0x65,0x67,0x0a,0x65,0x69,0x67,0x68,0x74,
+
0x0a,0x65,0x69,0x67,0x68,0x74,0x79,0x0a,0x65,0x69,0x74,0x68,0x65,0x72,0x0a,0x65,
+
0x6c,0x73,0x65,0x0a,0x65,0x6c,0x73,0x65,0x77,0x68,0x65,0x72,0x65,0x0a,0x65,0x6e,
+
0x64,0x0a,0x65,0x6e,0x64,0x69,0x6e,0x67,0x0a,0x65,0x6e,0x6f,0x75,0x67,0x68,0x0a,
+
0x65,0x73,0x70,0x65,0x63,0x69,0x61,0x6c,0x6c,0x79,0x0a,0x65,0x74,0x0a,0x65,0x74,
+
0x2d,0x61,0x6c,0x0a,0x65,0x74,0x63,0x0a,0x65,0x76,0x65,0x6e,0x0a,0x65,0x76,0x65,
+
0x72,0x0a,0x65,0x76,0x65,0x72,0x79,0x0a,0x65,0x76,0x65,0x72,0x79,0x62,0x6f,0x64,
+
0x79,0x0a,0x65,0x76,0x65,0x72,0x79,0x6f,0x6e,0x65,0x0a,0x65,0x76,0x65,0x72,0x79,
+
0x74,0x68,0x69,0x6e,0x67,0x0a,0x65,0x76,0x65,0x72,0x79,0x77,0x68,0x65,0x72,0x65,
+
0x0a,0x65,0x78,0x0a,0x65,0x78,0x63,0x65,0x70,0x74,0x0a,0x66,0x0a,0x66,0x61,0x72,
+
0x0a,0x66,0x65,0x77,0x0a,0x66,0x66,0x0a,0x66,0x69,0x66,0x74,0x68,0x0a,0x66,0x69,
+
0x72,0x73,0x74,0x0a,0x66,0x69,0x76,0x65,0x0a,0x66,0x69,0x78,0x0a,0x66,0x6f,0x6c,
+
0x6c,0x6f,0x77,0x65,0x64,0x0a,0x66,0x6f,0x6c,0x6c,0x6f,0x77,0x69,0x6e,0x67,0x0a,
+
0x66,0x6f,0x6c,0x6c,0x6f,0x77,0x73,0x0a,0x66,0x6f,0x72,0x0a,0x66,0x6f,0x72,0x6d,
+
0x65,0x72,0x0a,0x66,0x6f,0x72,0x6d,0x65,0x72,0x6c,0x79,0x0a,0x66,0x6f,0x72,0x74,
+
0x68,0x0a,0x66,0x6f,0x75,0x6e,0x64,0x0a,0x66,0x6f,0x75,0x72,0x0a,0x66,0x72,0x6f,
+
0x6d,0x0a,0x66,0x75,0x72,0x74,0x68,0x65,0x72,0x0a,0x66,0x75,0x72,0x74,0x68,0x65,
+
0x72,0x6d,0x6f,0x72,0x65,0x0a,0x67,0x0a,0x67,0x61,0x76,0x65,0x0a,0x67,0x65,0x74,
+
0x0a,0x67,0x65,0x74,0x73,0x0a,0x67,0x65,0x74,0x74,0x69,0x6e,0x67,0x0a,0x67,0x69,
+
0x76,0x65,0x0a,0x67,0x69,0x76,0x65,0x6e,0x0a,0x67,0x69,0x76,0x65,0x73,0x0a,0x67,
+
0x69,0x76,0x69,0x6e,0x67,0x0a,0x67,0x6f,0x0a,0x67,0x6f,0x65,0x73,0x0a,0x67,0x6f,
+
0x6e,0x65,0x0a,0x67,0x6f,0x74,0x0a,0x67,0x6f,0x74,0x74,0x65,0x6e,0x0a,0x68,0x0a,
+
0x68,0x61,0x64,0x0a,0x68,0x61,0x70,0x70,0x65,0x6e,0x73,0x0a,0x68,0x61,0x72,0x64,
+
0x6c,0x79,0x0a,0x68,0x61,0x73,0x0a,0x68,0x61,0x73,0x6e,0x27,0x74,0x0a,0x68,0x61,
+
0x76,0x65,0x0a,0x68,0x61,0x76,0x65,0x6e,0x27,0x74,0x0a,0x68,0x61,0x76,0x69,0x6e,
+
0x67,0x0a,0x68,0x65,0x0a,0x68,0x65,0x64,0x0a,0x68,0x65,0x6e,0x63,0x65,0x0a,0x68,
+
0x65,0x72,0x0a,0x68,0x65,0x72,0x65,0x0a,0x68,0x65,0x72,0x65,0x61,0x66,0x74,0x65,
+
0x72,0x0a,0x68,0x65,0x72,0x65,0x62,0x79,0x0a,0x68,0x65,0x72,0x65,0x69,0x6e,0x0a,
+
0x68,0x65,0x72,0x65,0x73,0x0a,0x68,0x65,0x72,0x65,0x75,0x70,0x6f,0x6e,0x0a,0x68,
+
0x65,0x72,0x73,0x0a,0x68,0x65,0x72,0x73,0x65,0x6c,0x66,0x0a,0x68,0x65,0x73,0x0a,
+
0x68,0x69,0x0a,0x68,0x69,0x64,0x0a,0x68,0x69,0x6d,0x0a,0x68,0x69,0x6d,0x73,0x65,
+
0x6c,0x66,0x0a,0x68,0x69,0x73,0x0a,0x68,0x69,0x74,0x68,0x65,0x72,0x0a,0x68,0x6f,
+
0x6d,0x65,0x0a,0x68,0x6f,0x77,0x0a,0x68,0x6f,0x77,0x62,0x65,0x69,0x74,0x0a,0x68,
+
0x6f,0x77,0x65,0x76,0x65,0x72,0x0a,0x68,0x75,0x6e,0x64,0x72,0x65,0x64,0x0a,0x69,
+
0x0a,0x69,0x64,0x0a,0x69,0x65,0x0a,0x69,0x66,0x0a,0x69,0x27,0x6c,0x6c,0x0a,0x69,
+
0x6d,0x0a,0x69,0x6d,0x6d,0x65,0x64,0x69,0x61,0x74,0x65,0x0a,0x69,0x6d,0x6d,0x65,
+
0x64,0x69,0x61,0x74,0x65,0x6c,0x79,0x0a,0x69,0x6d,0x70,0x6f,0x72,0x74,0x61,0x6e,
+
0x63,0x65,0x0a,0x69,0x6d,0x70,0x6f,0x72,0x74,0x61,0x6e,0x74,0x0a,0x69,0x6e,0x0a,
+
0x69,0x6e,0x63,0x0a,0x69,0x6e,0x64,0x65,0x65,0x64,0x0a,0x69,0x6e,0x64,0x65,0x78,
+
0x0a,0x69,0x6e,0x66,0x6f,0x72,0x6d,0x61,0x74,0x69,0x6f,0x6e,0x0a,0x69,0x6e,0x73,
+
0x74,0x65,0x61,0x64,0x0a,0x69,0x6e,0x74,0x6f,0x0a,0x69,0x6e,0x76,0x65,0x6e,0x74,
+
0x69,0x6f,0x6e,0x0a,0x69,0x6e,0x77,0x61,0x72,0x64,0x0a,0x69,0x73,0x0a,0x69,0x73,
+
0x6e,0x27,0x74,0x0a,0x69,0x74,0x0a,0x69,0x74,0x64,0x0a,0x69,0x74,0x27,0x6c,0x6c,
+
0x0a,0x69,0x74,0x73,0x0a,0x69,0x74,0x73,0x65,0x6c,0x66,0x0a,0x69,0x27,0x76,0x65,
+
0x0a,0x6a,0x0a,0x6a,0x75,0x73,0x74,0x0a,0x6b,0x0a,0x6b,0x65,0x65,0x70,0x0a,0x6b,
+
0x65,0x65,0x70,0x73,0x0a,0x6b,0x65,0x70,0x74,0x0a,0x6b,0x65,0x79,0x73,0x0a,0x6b,
+
0x67,0x0a,0x6b,0x6d,0x0a,0x6b,0x6e,0x6f,0x77,0x0a,0x6b,0x6e,0x6f,0x77,0x6e,0x0a,
+
0x6b,0x6e,0x6f,0x77,0x73,0x0a,0x6c,0x0a,0x6c,0x61,0x72,0x67,0x65,0x6c,0x79,0x0a,
+
0x6c,0x61,0x73,0x74,0x0a,0x6c,0x61,0x74,0x65,0x6c,0x79,0x0a,0x6c,0x61,0x74,0x65,
+
0x72,0x0a,0x6c,0x61,0x74,0x74,0x65,0x72,0x0a,0x6c,0x61,0x74,0x74,0x65,0x72,0x6c,
+
0x79,0x0a,0x6c,0x65,0x61,0x73,0x74,0x0a,0x6c,0x65,0x73,0x73,0x0a,0x6c,0x65,0x73,
+
0x74,0x0a,0x6c,0x65,0x74,0x0a,0x6c,0x65,0x74,0x73,0x0a,0x6c,0x69,0x6b,0x65,0x0a,
+
0x6c,0x69,0x6b,0x65,0x64,0x0a,0x6c,0x69,0x6b,0x65,0x6c,0x79,0x0a,0x6c,0x69,0x6e,
+
0x65,0x0a,0x6c,0x69,0x74,0x74,0x6c,0x65,0x0a,0x27,0x6c,0x6c,0x0a,0x6c,0x6f,0x6f,
+
0x6b,0x0a,0x6c,0x6f,0x6f,0x6b,0x69,0x6e,0x67,0x0a,0x6c,0x6f,0x6f,0x6b,0x73,0x0a,
+
0x6c,0x74,0x64,0x0a,0x6d,0x0a,0x6d,0x61,0x64,0x65,0x0a,0x6d,0x61,0x69,0x6e,0x6c,
+
0x79,0x0a,0x6d,0x61,0x6b,0x65,0x0a,0x6d,0x61,0x6b,0x65,0x73,0x0a,0x6d,0x61,0x6e,
+
0x79,0x0a,0x6d,0x61,0x79,0x0a,0x6d,0x61,0x79,0x62,0x65,0x0a,0x6d,0x65,0x0a,0x6d,
+
0x65,0x61,0x6e,0x0a,0x6d,0x65,0x61,0x6e,0x73,0x0a,0x6d,0x65,0x61,0x6e,0x74,0x69,
+
0x6d,0x65,0x0a,0x6d,0x65,0x61,0x6e,0x77,0x68,0x69,0x6c,0x65,0x0a,0x6d,0x65,0x72,
+
0x65,0x6c,0x79,0x0a,0x6d,0x67,0x0a,0x6d,0x69,0x67,0x68,0x74,0x0a,0x6d,0x69,0x6c,
+
0x6c,0x69,0x6f,0x6e,0x0a,0x6d,0x69,0x73,0x73,0x0a,0x6d,0x6c,0x0a,0x6d,0x6f,0x72,
+
0x65,0x0a,0x6d,0x6f,0x72,0x65,0x6f,0x76,0x65,0x72,0x0a,0x6d,0x6f,0x73,0x74,0x0a,
+
0x6d,0x6f,0x73,0x74,0x6c,0x79,0x0a,0x6d,0x72,0x0a,0x6d,0x72,0x73,0x0a,0x6d,0x75,
+
0x63,0x68,0x0a,0x6d,0x75,0x67,0x0a,0x6d,0x75,0x73,0x74,0x0a,0x6d,0x79,0x0a,0x6d,
+
0x79,0x73,0x65,0x6c,0x66,0x0a,0x6e,0x0a,0x6e,0x61,0x0a,0x6e,0x61,0x6d,0x65,0x0a,
+
0x6e,0x61,0x6d,0x65,0x6c,0x79,0x0a,0x6e,0x61,0x79,0x0a,0x6e,0x64,0x0a,0x6e,0x65,
+
0x61,0x72,0x0a,0x6e,0x65,0x61,0x72,0x6c,0x79,0x0a,0x6e,0x65,0x63,0x65,0x73,0x73,
+
0x61,0x72,0x69,0x6c,0x79,0x0a,0x6e,0x65,0x63,0x65,0x73,0x73,0x61,0x72,0x79,0x0a,
+
0x6e,0x65,0x65,0x64,0x0a,0x6e,0x65,0x65,0x64,0x73,0x0a,0x6e,0x65,0x69,0x74,0x68,
+
0x65,0x72,0x0a,0x6e,0x65,0x76,0x65,0x72,0x0a,0x6e,0x65,0x76,0x65,0x72,0x74,0x68,
+
0x65,0x6c,0x65,0x73,0x73,0x0a,0x6e,0x65,0x77,0x0a,0x6e,0x65,0x78,0x74,0x0a,0x6e,
+
0x69,0x6e,0x65,0x0a,0x6e,0x69,0x6e,0x65,0x74,0x79,0x0a,0x6e,0x6f,0x0a,0x6e,0x6f,
+
0x62,0x6f,0x64,0x79,0x0a,0x6e,0x6f,0x6e,0x0a,0x6e,0x6f,0x6e,0x65,0x0a,0x6e,0x6f,
+
0x6e,0x65,0x74,0x68,0x65,0x6c,0x65,0x73,0x73,0x0a,0x6e,0x6f,0x6f,0x6e,0x65,0x0a,
+
0x6e,0x6f,0x72,0x0a,0x6e,0x6f,0x72,0x6d,0x61,0x6c,0x6c,0x79,0x0a,0x6e,0x6f,0x73,
+
0x0a,0x6e,0x6f,0x74,0x0a,0x6e,0x6f,0x74,0x65,0x64,0x0a,0x6e,0x6f,0x74,0x68,0x69,
+
0x6e,0x67,0x0a,0x6e,0x6f,0x77,0x0a,0x6e,0x6f,0x77,0x68,0x65,0x72,0x65,0x0a,0x6f,
+
0x0a,0x6f,0x62,0x74,0x61,0x69,0x6e,0x0a,0x6f,0x62,0x74,0x61,0x69,0x6e,0x65,0x64,
+
0x0a,0x6f,0x62,0x76,0x69,0x6f,0x75,0x73,0x6c,0x79,0x0a,0x6f,0x66,0x0a,0x6f,0x66,
+
0x66,0x0a,0x6f,0x66,0x74,0x65,0x6e,0x0a,0x6f,0x68,0x0a,0x6f,0x6b,0x0a,0x6f,0x6b,
+
0x61,0x79,0x0a,0x6f,0x6c,0x64,0x0a,0x6f,0x6d,0x69,0x74,0x74,0x65,0x64,0x0a,0x6f,
+
0x6e,0x0a,0x6f,0x6e,0x63,0x65,0x0a,0x6f,0x6e,0x65,0x0a,0x6f,0x6e,0x65,0x73,0x0a,
+
0x6f,0x6e,0x6c,0x79,0x0a,0x6f,0x6e,0x74,0x6f,0x0a,0x6f,0x72,0x0a,0x6f,0x72,0x64,
+
0x0a,0x6f,0x74,0x68,0x65,0x72,0x0a,0x6f,0x74,0x68,0x65,0x72,0x73,0x0a,0x6f,0x74,
+
0x68,0x65,0x72,0x77,0x69,0x73,0x65,0x0a,0x6f,0x75,0x67,0x68,0x74,0x0a,0x6f,0x75,
+
0x72,0x0a,0x6f,0x75,0x72,0x73,0x0a,0x6f,0x75,0x72,0x73,0x65,0x6c,0x76,0x65,0x73,
+
0x0a,0x6f,0x75,0x74,0x0a,0x6f,0x75,0x74,0x73,0x69,0x64,0x65,0x0a,0x6f,0x76,0x65,
+
0x72,0x0a,0x6f,0x76,0x65,0x72,0x61,0x6c,0x6c,0x0a,0x6f,0x77,0x69,0x6e,0x67,0x0a,
+
0x6f,0x77,0x6e,0x0a,0x70,0x0a,0x70,0x61,0x67,0x65,0x0a,0x70,0x61,0x67,0x65,0x73,
+
0x0a,0x70,0x61,0x72,0x74,0x0a,0x70,0x61,0x72,0x74,0x69,0x63,0x75,0x6c,0x61,0x72,
+
0x0a,0x70,0x61,0x72,0x74,0x69,0x63,0x75,0x6c,0x61,0x72,0x6c,0x79,0x0a,0x70,0x61,
+
0x73,0x74,0x0a,0x70,0x65,0x72,0x0a,0x70,0x65,0x72,0x68,0x61,0x70,0x73,0x0a,0x70,
+
0x6c,0x61,0x63,0x65,0x64,0x0a,0x70,0x6c,0x65,0x61,0x73,0x65,0x0a,0x70,0x6c,0x75,
+
0x73,0x0a,0x70,0x6f,0x6f,0x72,0x6c,0x79,0x0a,0x70,0x6f,0x73,0x73,0x69,0x62,0x6c,
+
0x65,0x0a,0x70,0x6f,0x73,0x73,0x69,0x62,0x6c,0x79,0x0a,0x70,0x6f,0x74,0x65,0x6e,
+
0x74,0x69,0x61,0x6c,0x6c,0x79,0x0a,0x70,0x70,0x0a,0x70,0x72,0x65,0x64,0x6f,0x6d,
+
0x69,0x6e,0x61,0x6e,0x74,0x6c,0x79,0x0a,0x70,0x72,0x65,0x73,0x65,0x6e,0x74,0x0a,
+
0x70,0x72,0x65,0x76,0x69,0x6f,0x75,0x73,0x6c,0x79,0x0a,0x70,0x72,0x69,0x6d,0x61,
+
0x72,0x69,0x6c,0x79,0x0a,0x70,0x72,0x6f,0x62,0x61,0x62,0x6c,0x79,0x0a,0x70,0x72,
+
0x6f,0x6d,0x70,0x74,0x6c,0x79,0x0a,0x70,0x72,0x6f,0x75,0x64,0x0a,0x70,0x72,0x6f,
+
0x76,0x69,0x64,0x65,0x73,0x0a,0x70,0x75,0x74,0x0a,0x71,0x0a,0x71,0x75,0x65,0x0a,
+
0x71,0x75,0x69,0x63,0x6b,0x6c,0x79,0x0a,0x71,0x75,0x69,0x74,0x65,0x0a,0x71,0x76,
+
0x0a,0x72,0x0a,0x72,0x61,0x6e,0x0a,0x72,0x61,0x74,0x68,0x65,0x72,0x0a,0x72,0x64,
+
0x0a,0x72,0x65,0x0a,0x72,0x65,0x61,0x64,0x69,0x6c,0x79,0x0a,0x72,0x65,0x61,0x6c,
+
0x6c,0x79,0x0a,0x72,0x65,0x63,0x65,0x6e,0x74,0x0a,0x72,0x65,0x63,0x65,0x6e,0x74,
+
0x6c,0x79,0x0a,0x72,0x65,0x66,0x0a,0x72,0x65,0x66,0x73,0x0a,0x72,0x65,0x67,0x61,
+
0x72,0x64,0x69,0x6e,0x67,0x0a,0x72,0x65,0x67,0x61,0x72,0x64,0x6c,0x65,0x73,0x73,
+
0x0a,0x72,0x65,0x67,0x61,0x72,0x64,0x73,0x0a,0x72,0x65,0x6c,0x61,0x74,0x65,0x64,
+
0x0a,0x72,0x65,0x6c,0x61,0x74,0x69,0x76,0x65,0x6c,0x79,0x0a,0x72,0x65,0x73,0x65,
+
0x61,0x72,0x63,0x68,0x0a,0x72,0x65,0x73,0x70,0x65,0x63,0x74,0x69,0x76,0x65,0x6c,
+
0x79,0x0a,0x72,0x65,0x73,0x75,0x6c,0x74,0x65,0x64,0x0a,0x72,0x65,0x73,0x75,0x6c,
+
0x74,0x69,0x6e,0x67,0x0a,0x72,0x65,0x73,0x75,0x6c,0x74,0x73,0x0a,0x72,0x69,0x67,
+
0x68,0x74,0x0a,0x72,0x75,0x6e,0x0a,0x73,0x0a,0x73,0x61,0x69,0x64,0x0a,0x73,0x61,
+
0x6d,0x65,0x0a,0x73,0x61,0x77,0x0a,0x73,0x61,0x79,0x0a,0x73,0x61,0x79,0x69,0x6e,
+
0x67,0x0a,0x73,0x61,0x79,0x73,0x0a,0x73,0x65,0x63,0x0a,0x73,0x65,0x63,0x74,0x69,
+
0x6f,0x6e,0x0a,0x73,0x65,0x65,0x0a,0x73,0x65,0x65,0x69,0x6e,0x67,0x0a,0x73,0x65,
+
0x65,0x6d,0x0a,0x73,0x65,0x65,0x6d,0x65,0x64,0x0a,0x73,0x65,0x65,0x6d,0x69,0x6e,
+
0x67,0x0a,0x73,0x65,0x65,0x6d,0x73,0x0a,0x73,0x65,0x65,0x6e,0x0a,0x73,0x65,0x6c,
+
0x66,0x0a,0x73,0x65,0x6c,0x76,0x65,0x73,0x0a,0x73,0x65,0x6e,0x74,0x0a,0x73,0x65,
+
0x76,0x65,0x6e,0x0a,0x73,0x65,0x76,0x65,0x72,0x61,0x6c,0x0a,0x73,0x68,0x61,0x6c,
+
0x6c,0x0a,0x73,0x68,0x65,0x0a,0x73,0x68,0x65,0x64,0x0a,0x73,0x68,0x65,0x27,0x6c,
+
0x6c,0x0a,0x73,0x68,0x65,0x73,0x0a,0x73,0x68,0x6f,0x75,0x6c,0x64,0x0a,0x73,0x68,
+
0x6f,0x75,0x6c,0x64,0x6e,0x27,0x74,0x0a,0x73,0x68,0x6f,0x77,0x0a,0x73,0x68,0x6f,
+
0x77,0x65,0x64,0x0a,0x73,0x68,0x6f,0x77,0x6e,0x0a,0x73,0x68,0x6f,0x77,0x6e,0x73,
+
0x0a,0x73,0x68,0x6f,0x77,0x73,0x0a,0x73,0x69,0x67,0x6e,0x69,0x66,0x69,0x63,0x61,
+
0x6e,0x74,0x0a,0x73,0x69,0x67,0x6e,0x69,0x66,0x69,0x63,0x61,0x6e,0x74,0x6c,0x79,
+
0x0a,0x73,0x69,0x6d,0x69,0x6c,0x61,0x72,0x0a,0x73,0x69,0x6d,0x69,0x6c,0x61,0x72,
+
0x6c,0x79,0x0a,0x73,0x69,0x6e,0x63,0x65,0x0a,0x73,0x69,0x78,0x0a,0x73,0x6c,0x69,
+
0x67,0x68,0x74,0x6c,0x79,0x0a,0x73,0x6f,0x0a,0x73,0x6f,0x6d,0x65,0x0a,0x73,0x6f,
+
0x6d,0x65,0x62,0x6f,0x64,0x79,0x0a,0x73,0x6f,0x6d,0x65,0x68,0x6f,0x77,0x0a,0x73,
+
0x6f,0x6d,0x65,0x6f,0x6e,0x65,0x0a,0x73,0x6f,0x6d,0x65,0x74,0x68,0x61,0x6e,0x0a,
+
0x73,0x6f,0x6d,0x65,0x74,0x68,0x69,0x6e,0x67,0x0a,0x73,0x6f,0x6d,0x65,0x74,0x69,
+
0x6d,0x65,0x0a,0x73,0x6f,0x6d,0x65,0x74,0x69,0x6d,0x65,0x73,0x0a,0x73,0x6f,0x6d,
+
0x65,0x77,0x68,0x61,0x74,0x0a,0x73,0x6f,0x6d,0x65,0x77,0x68,0x65,0x72,0x65,0x0a,
+
0x73,0x6f,0x6f,0x6e,0x0a,0x73,0x6f,0x72,0x72,0x79,0x0a,0x73,0x70,0x65,0x63,0x69,
+
0x66,0x69,0x63,0x61,0x6c,0x6c,0x79,0x0a,0x73,0x70,0x65,0x63,0x69,0x66,0x69,0x65,
+
0x64,0x0a,0x73,0x70,0x65,0x63,0x69,0x66,0x79,0x0a,0x73,0x70,0x65,0x63,0x69,0x66,
+
0x79,0x69,0x6e,0x67,0x0a,0x73,0x74,0x61,0x74,0x65,0x0a,0x73,0x74,0x61,0x74,0x65,
+
0x73,0x0a,0x73,0x74,0x69,0x6c,0x6c,0x0a,0x73,0x74,0x6f,0x70,0x0a,0x73,0x74,0x72,
+
0x6f,0x6e,0x67,0x6c,0x79,0x0a,0x73,0x75,0x62,0x0a,0x73,0x75,0x62,0x73,0x74,0x61,
+
0x6e,0x74,0x69,0x61,0x6c,0x6c,0x79,0x0a,0x73,0x75,0x63,0x63,0x65,0x73,0x73,0x66,
+
0x75,0x6c,0x6c,0x79,0x0a,0x73,0x75,0x63,0x68,0x0a,0x73,0x75,0x66,0x66,0x69,0x63,
+
0x69,0x65,0x6e,0x74,0x6c,0x79,0x0a,0x73,0x75,0x67,0x67,0x65,0x73,0x74,0x0a,0x73,
+
0x75,0x70,0x0a,0x73,0x75,0x72,0x65,0x0a,0x09,0x74,0x0a,0x74,0x61,0x6b,0x65,0x0a,
+
0x74,0x61,0x6b,0x65,0x6e,0x0a,0x74,0x61,0x6b,0x69,0x6e,0x67,0x0a,0x74,0x65,0x6c,
+
0x6c,0x0a,0x74,0x65,0x6e,0x64,0x73,0x0a,0x74,0x68,0x0a,0x74,0x68,0x61,0x6e,0x0a,
+
0x74,0x68,0x61,0x6e,0x6b,0x0a,0x74,0x68,0x61,0x6e,0x6b,0x73,0x0a,0x74,0x68,0x61,
+
0x6e,0x78,0x0a,0x74,0x68,0x61,0x74,0x0a,0x74,0x68,0x61,0x74,0x27,0x6c,0x6c,0x0a,
+
0x74,0x68,0x61,0x74,0x73,0x0a,0x74,0x68,0x61,0x74,0x27,0x76,0x65,0x0a,0x74,0x68,
+
0x65,0x0a,0x74,0x68,0x65,0x69,0x72,0x0a,0x74,0x68,0x65,0x69,0x72,0x73,0x0a,0x74,
+
0x68,0x65,0x6d,0x0a,0x74,0x68,0x65,0x6d,0x73,0x65,0x6c,0x76,0x65,0x73,0x0a,0x74,
+
0x68,0x65,0x6e,0x0a,0x74,0x68,0x65,0x6e,0x63,0x65,0x0a,0x74,0x68,0x65,0x72,0x65,
+
0x0a,0x74,0x68,0x65,0x72,0x65,0x61,0x66,0x74,0x65,0x72,0x0a,0x74,0x68,0x65,0x72,
+
0x65,0x62,0x79,0x0a,0x74,0x68,0x65,0x72,0x65,0x64,0x0a,0x74,0x68,0x65,0x72,0x65,
+
0x66,0x6f,0x72,0x65,0x0a,0x74,0x68,0x65,0x72,0x65,0x69,0x6e,0x0a,0x74,0x68,0x65,
+
0x72,0x65,0x27,0x6c,0x6c,0x0a,0x74,0x68,0x65,0x72,0x65,0x6f,0x66,0x0a,0x74,0x68,
+
0x65,0x72,0x65,0x72,0x65,0x0a,0x74,0x68,0x65,0x72,0x65,0x73,0x0a,0x74,0x68,0x65,
+
0x72,0x65,0x74,0x6f,0x0a,0x74,0x68,0x65,0x72,0x65,0x75,0x70,0x6f,0x6e,0x0a,0x74,
+
0x68,0x65,0x72,0x65,0x27,0x76,0x65,0x0a,0x74,0x68,0x65,0x73,0x65,0x0a,0x74,0x68,
+
0x65,0x79,0x0a,0x74,0x68,0x65,0x79,0x64,0x0a,0x74,0x68,0x65,0x79,0x27,0x6c,0x6c,
+
0x0a,0x74,0x68,0x65,0x79,0x72,0x65,0x0a,0x74,0x68,0x65,0x79,0x27,0x76,0x65,0x0a,
+
0x74,0x68,0x69,0x6e,0x6b,0x0a,0x74,0x68,0x69,0x73,0x0a,0x74,0x68,0x6f,0x73,0x65,
+
0x0a,0x74,0x68,0x6f,0x75,0x0a,0x74,0x68,0x6f,0x75,0x67,0x68,0x0a,0x74,0x68,0x6f,
+
0x75,0x67,0x68,0x68,0x0a,0x74,0x68,0x6f,0x75,0x73,0x61,0x6e,0x64,0x0a,0x74,0x68,
+
0x72,0x6f,0x75,0x67,0x0a,0x74,0x68,0x72,0x6f,0x75,0x67,0x68,0x0a,0x74,0x68,0x72,
+
0x6f,0x75,0x67,0x68,0x6f,0x75,0x74,0x0a,0x74,0x68,0x72,0x75,0x0a,0x74,0x68,0x75,
+
0x73,0x0a,0x74,0x69,0x6c,0x0a,0x74,0x69,0x70,0x0a,0x74,0x6f,0x0a,0x74,0x6f,0x67,
+
0x65,0x74,0x68,0x65,0x72,0x0a,0x74,0x6f,0x6f,0x0a,0x74,0x6f,0x6f,0x6b,0x0a,0x74,
+
0x6f,0x77,0x61,0x72,0x64,0x0a,0x74,0x6f,0x77,0x61,0x72,0x64,0x73,0x0a,0x74,0x72,
+
0x69,0x65,0x64,0x0a,0x74,0x72,0x69,0x65,0x73,0x0a,0x74,0x72,0x75,0x6c,0x79,0x0a,
+
0x74,0x72,0x79,0x0a,0x74,0x72,0x79,0x69,0x6e,0x67,0x0a,0x74,0x73,0x0a,0x74,0x77,
+
0x69,0x63,0x65,0x0a,0x74,0x77,0x6f,0x0a,0x75,0x0a,0x75,0x6e,0x0a,0x75,0x6e,0x64,
+
0x65,0x72,0x0a,0x75,0x6e,0x66,0x6f,0x72,0x74,0x75,0x6e,0x61,0x74,0x65,0x6c,0x79,
+
0x0a,0x75,0x6e,0x6c,0x65,0x73,0x73,0x0a,0x75,0x6e,0x6c,0x69,0x6b,0x65,0x0a,0x75,
+
0x6e,0x6c,0x69,0x6b,0x65,0x6c,0x79,0x0a,0x75,0x6e,0x74,0x69,0x6c,0x0a,0x75,0x6e,
+
0x74,0x6f,0x0a,0x75,0x70,0x0a,0x75,0x70,0x6f,0x6e,0x0a,0x75,0x70,0x73,0x0a,0x75,
+
0x73,0x0a,0x75,0x73,0x65,0x0a,0x75,0x73,0x65,0x64,0x0a,0x75,0x73,0x65,0x66,0x75,
+
0x6c,0x0a,0x75,0x73,0x65,0x66,0x75,0x6c,0x6c,0x79,0x0a,0x75,0x73,0x65,0x66,0x75,
+
0x6c,0x6e,0x65,0x73,0x73,0x0a,0x75,0x73,0x65,0x73,0x0a,0x75,0x73,0x69,0x6e,0x67,
+
0x0a,0x75,0x73,0x75,0x61,0x6c,0x6c,0x79,0x0a,0x76,0x0a,0x76,0x61,0x6c,0x75,0x65,
+
0x0a,0x76,0x61,0x72,0x69,0x6f,0x75,0x73,0x0a,0x27,0x76,0x65,0x0a,0x76,0x65,0x72,
+
0x79,0x0a,0x76,0x69,0x61,0x0a,0x76,0x69,0x7a,0x0a,0x76,0x6f,0x6c,0x0a,0x76,0x6f,
+
0x6c,0x73,0x0a,0x76,0x73,0x0a,0x77,0x0a,0x77,0x61,0x6e,0x74,0x0a,0x77,0x61,0x6e,
+
0x74,0x73,0x0a,0x77,0x61,0x73,0x0a,0x77,0x61,0x73,0x6e,0x27,0x74,0x0a,0x77,0x61,
+
0x79,0x0a,0x77,0x65,0x0a,0x77,0x65,0x64,0x0a,0x77,0x65,0x6c,0x63,0x6f,0x6d,0x65,
+
0x0a,0x77,0x65,0x27,0x6c,0x6c,0x0a,0x77,0x65,0x6e,0x74,0x0a,0x77,0x65,0x72,0x65,
+
0x0a,0x77,0x65,0x72,0x65,0x6e,0x27,0x74,0x0a,0x77,0x65,0x27,0x76,0x65,0x0a,0x77,
+
0x68,0x61,0x74,0x0a,0x77,0x68,0x61,0x74,0x65,0x76,0x65,0x72,0x0a,0x77,0x68,0x61,
+
0x74,0x27,0x6c,0x6c,0x0a,0x77,0x68,0x61,0x74,0x73,0x0a,0x77,0x68,0x65,0x6e,0x0a,
+
0x77,0x68,0x65,0x6e,0x63,0x65,0x0a,0x77,0x68,0x65,0x6e,0x65,0x76,0x65,0x72,0x0a,
+
0x77,0x68,0x65,0x72,0x65,0x0a,0x77,0x68,0x65,0x72,0x65,0x61,0x66,0x74,0x65,0x72,
+
0x0a,0x77,0x68,0x65,0x72,0x65,0x61,0x73,0x0a,0x77,0x68,0x65,0x72,0x65,0x62,0x79,
+
0x0a,0x77,0x68,0x65,0x72,0x65,0x69,0x6e,0x0a,0x77,0x68,0x65,0x72,0x65,0x73,0x0a,
+
0x77,0x68,0x65,0x72,0x65,0x75,0x70,0x6f,0x6e,0x0a,0x77,0x68,0x65,0x72,0x65,0x76,
+
0x65,0x72,0x0a,0x77,0x68,0x65,0x74,0x68,0x65,0x72,0x0a,0x77,0x68,0x69,0x63,0x68,
+
0x0a,0x77,0x68,0x69,0x6c,0x65,0x0a,0x77,0x68,0x69,0x6d,0x0a,0x77,0x68,0x69,0x74,
+
0x68,0x65,0x72,0x0a,0x77,0x68,0x6f,0x0a,0x77,0x68,0x6f,0x64,0x0a,0x77,0x68,0x6f,
+
0x65,0x76,0x65,0x72,0x0a,0x77,0x68,0x6f,0x6c,0x65,0x0a,0x77,0x68,0x6f,0x27,0x6c,
+
0x6c,0x0a,0x77,0x68,0x6f,0x6d,0x0a,0x77,0x68,0x6f,0x6d,0x65,0x76,0x65,0x72,0x0a,
+
0x77,0x68,0x6f,0x73,0x0a,0x77,0x68,0x6f,0x73,0x65,0x0a,0x77,0x68,0x79,0x0a,0x77,
+
0x69,0x64,0x65,0x6c,0x79,0x0a,0x77,0x69,0x6c,0x6c,0x69,0x6e,0x67,0x0a,0x77,0x69,
+
0x73,0x68,0x0a,0x77,0x69,0x74,0x68,0x0a,0x77,0x69,0x74,0x68,0x69,0x6e,0x0a,0x77,
+
0x69,0x74,0x68,0x6f,0x75,0x74,0x0a,0x77,0x6f,0x6e,0x27,0x74,0x0a,0x77,0x6f,0x72,
+
0x64,0x73,0x0a,0x77,0x6f,0x72,0x6c,0x64,0x0a,0x77,0x6f,0x75,0x6c,0x64,0x0a,0x77,
+
0x6f,0x75,0x6c,0x64,0x6e,0x27,0x74,0x0a,0x77,0x77,0x77,0x0a,0x78,0x0a,0x79,0x0a,
+
0x79,0x65,0x73,0x0a,0x79,0x65,0x74,0x0a,0x79,0x6f,0x75,0x0a,0x79,0x6f,0x75,0x64,
+
0x0a,0x79,0x6f,0x75,0x27,0x6c,0x6c,0x0a,0x79,0x6f,0x75,0x72,0x0a,0x79,0x6f,0x75,
+
0x72,0x65,0x0a,0x79,0x6f,0x75,0x72,0x73,0x0a,0x79,0x6f,0x75,0x72,0x73,0x65,0x6c,
+
0x66,0x0a,0x79,0x6f,0x75,0x72,0x73,0x65,0x6c,0x76,0x65,0x73,0x0a,0x79,0x6f,0x75,
+ 0x27,0x76,0x65,0x0a,0x7a,0x0a,0x7a,0x65,0x72,0x6f,0x0a
+ };
+
+static const unsigned char stopwords_fra[]={
+
0x61,0x6c,0x6f,0x72,0x73,0x0a,0x61,0x75,0x0a,0x61,0x75,0x63,0x75,0x6e,0x73,0x0a,
+
0x61,0x75,0x73,0x73,0x69,0x0a,0x61,0x75,0x74,0x72,0x65,0x0a,0x61,0x76,0x61,0x6e,
+
0x74,0x0a,0x61,0x76,0x65,0x63,0x0a,0x61,0x76,0x6f,0x69,0x72,0x0a,0x62,0x6f,0x6e,
+
0x0a,0x63,0x61,0x72,0x0a,0x63,0x65,0x0a,0x63,0x65,0x6c,0x61,0x0a,0x63,0x65,0x73,
+
0x0a,0x63,0x65,0x75,0x78,0x0a,0x63,0x68,0x61,0x71,0x75,0x65,0x0a,0x63,0x69,0x0a,
+
0x63,0x6f,0x6d,0x6d,0x65,0x0a,0x63,0x6f,0x6d,0x6d,0x65,0x6e,0x74,0x0a,0x64,0x61,
+
0x6e,0x73,0x0a,0x64,0x65,0x73,0x0a,0x64,0x75,0x0a,0x64,0x65,0x64,0x61,0x6e,0x73,
+
0x0a,0x64,0x65,0x68,0x6f,0x72,0x73,0x0a,0x64,0x65,0x70,0x75,0x69,0x73,0x0a,0x64,
+
0x65,0x75,0x78,0x0a,0x64,0x65,0x76,0x72,0x61,0x69,0x74,0x0a,0x64,0x6f,0x69,0x74,
+
0x0a,0x64,0x6f,0x6e,0x63,0x0a,0x64,0x6f,0x73,0x0a,0x64,0x72,0x6f,0x69,0x74,0x65,
+
0x0a,0x64,0xc3,0xa9,0x62,0x75,0x74,0x0a,0x65,0x6c,0x6c,0x65,0x0a,0x65,0x6c,0x6c,
+
0x65,0x73,0x0a,0x65,0x6e,0x0a,0x65,0x6e,0x63,0x6f,0x72,0x65,0x0a,0x65,0x73,0x73,
+
0x61,0x69,0x0a,0x65,0x73,0x74,0x0a,0x65,0x74,0x0a,0x65,0x75,0x0a,0x66,0x61,0x69,
+
0x74,0x0a,0x66,0x61,0x69,0x74,0x65,0x73,0x0a,0x66,0x6f,0x69,0x73,0x0a,0x66,0x6f,
+
0x6e,0x74,0x0a,0x66,0x6f,0x72,0x63,0x65,0x0a,0x68,0x61,0x75,0x74,0x0a,0x68,0x6f,
+
0x72,0x73,0x0a,0x69,0x63,0x69,0x0a,0x69,0x6c,0x0a,0x69,0x6c,0x73,0x0a,0x6a,0x65,
+
0x0a,0x6c,0x61,0x0a,0x6c,0x65,0x0a,0x6c,0x65,0x73,0x0a,0x6c,0x65,0x75,0x72,0x0a,
+
0x6c,0xc3,0xa0,0x0a,0x6d,0x61,0x0a,0x6d,0x61,0x69,0x6e,0x74,0x65,0x6e,0x61,0x6e,
+
0x74,0x0a,0x6d,0x61,0x69,0x73,0x0a,0x6d,0x65,0x73,0x0a,0x6d,0x69,0x6e,0x65,0x0a,
+
0x6d,0x6f,0x69,0x6e,0x73,0x0a,0x6d,0x6f,0x6e,0x0a,0x6d,0x6f,0x74,0x0a,0x6d,0xc3,
+
0xaa,0x6d,0x65,0x0a,0x6e,0x69,0x0a,0x6e,0x6f,0x6d,0x6d,0xc3,0xa9,0x73,0x0a,0x6e,
+
0x6f,0x74,0x72,0x65,0x0a,0x6e,0x6f,0x75,0x73,0x0a,0x6e,0x6f,0x75,0x76,0x65,0x61,
+
0x75,0x78,0x0a,0x6f,0x75,0x0a,0x6f,0xc3,0xb9,0x0a,0x70,0x61,0x72,0x0a,0x70,0x61,
+
0x72,0x63,0x65,0x0a,0x70,0x61,0x72,0x6f,0x6c,0x65,0x0a,0x70,0x61,0x73,0x0a,0x70,
+
0x65,0x72,0x73,0x6f,0x6e,0x6e,0x65,0x73,0x0a,0x70,0x65,0x75,0x74,0x0a,0x70,0x65,
+
0x75,0x0a,0x70,0x69,0xc3,0xa8,0x63,0x65,0x0a,0x70,0x6c,0x75,0x70,0x61,0x72,0x74,
+
0x0a,0x70,0x6f,0x75,0x72,0x0a,0x70,0x6f,0x75,0x72,0x71,0x75,0x6f,0x69,0x0a,0x71,
+
0x75,0x61,0x6e,0x64,0x0a,0x71,0x75,0x65,0x0a,0x71,0x75,0x65,0x6c,0x0a,0x71,0x75,
+
0x65,0x6c,0x6c,0x65,0x0a,0x71,0x75,0x65,0x6c,0x6c,0x65,0x73,0x0a,0x71,0x75,0x65,
+
0x6c,0x73,0x0a,0x71,0x75,0x69,0x0a,0x73,0x61,0x0a,0x73,0x61,0x6e,0x73,0x0a,0x73,
+
0x65,0x73,0x0a,0x73,0x65,0x75,0x6c,0x65,0x6d,0x65,0x6e,0x74,0x0a,0x73,0x69,0x0a,
+
0x73,0x69,0x65,0x6e,0x0a,0x73,0x6f,0x6e,0x0a,0x73,0x6f,0x6e,0x74,0x0a,0x73,0x6f,
+
0x75,0x73,0x0a,0x73,0x6f,0x79,0x65,0x7a,0x0a,0x73,0x75,0x72,0x0a,0x74,0x61,0x0a,
+
0x74,0x61,0x6e,0x64,0x69,0x73,0x0a,0x74,0x65,0x6c,0x6c,0x65,0x6d,0x65,0x6e,0x74,
+
0x0a,0x74,0x65,0x6c,0x73,0x0a,0x74,0x65,0x73,0x0a,0x74,0x6f,0x6e,0x0a,0x74,0x6f,
+
0x75,0x73,0x0a,0x74,0x6f,0x75,0x74,0x0a,0x74,0x72,0x6f,0x70,0x0a,0x74,0x72,0xc3,
+
0xa8,0x73,0x0a,0x74,0x75,0x0a,0x76,0x61,0x6c,0x65,0x75,0x72,0x0a,0x76,0x6f,0x69,
+
0x65,0x0a,0x76,0x6f,0x69,0x65,0x6e,0x74,0x0a,0x76,0x6f,0x6e,0x74,0x0a,0x76,0x6f,
+
0x74,0x72,0x65,0x0a,0x76,0x6f,0x75,0x73,0x0a,0x76,0x75,0x0a,0xc3,0xa7,0x61,0x0a,
+
0xc3,0xa9,0x74,0x61,0x69,0x65,0x6e,0x74,0x0a,0xc3,0xa9,0x74,0x61,0x74,0x0a,0xc3,
+
0xa9,0x74,0x69,0x6f,0x6e,0x73,0x0a,0xc3,0xa9,0x74,0xc3,0xa9,0x0a,0xc3,0xaa,0x74,
+ 0x72,0x65,0x0a
+ };
+
+static const unsigned char stopwords_he[]={
+
0xd7,0xa9,0xd7,0x9c,0x0d,0x0a,0xd7,0x90,0xd7,0xaa,0x0d,0x0a,0xd7,0xa2,0xd7,0x9c,
+
0x0d,0x0a,0xd7,0x9c,0xd7,0x90,0x0d,0x0a,0xd7,0x9b,0xd7,0x99,0x0d,0x0a,0xd7,0xa2,
+
0xd7,0x9d,0x0d,0x0a,0xd7,0x94,0xd7,0x95,0xd7,0x90,0x0d,0x0a,0xd7,0x92,0xd7,0x9d,
+
0x0d,0x0a,0xd7,0x91,0x0d,0x0a,0xd7,0x96,0xd7,0x94,0x0d,0x0a,0xd7,0x94,0xd7,0x99,
+
0xd7,0x90,0x0d,0x0a,0xd7,0x9b,0xd7,0x9c,0x0d,0x0a,0xd7,0x99,0xd7,0x95,0xd7,0xaa,
+
0xd7,0xa8,0x0d,0x0a,0xd7,0x90,0xd7,0x95,0x0d,0x0a,0xd7,0x90,0xd7,0x91,0xd7,0x9c,
+
0x0d,0x0a,0xd7,0x91,0xd7,0x99,0xd7,0x9f,0x0d,0x0a,0xd7,0x94,0xd7,0x99,0xd7,0x94,
+
0x0d,0x0a,0xd7,0x90,0xd7,0x9d,0x0d,0x0a,0xd7,0x9e,0xd7,0x99,0xd7,0x9c,0xd7,0x99,
+
0xd7,0x95,0xd7,0x9f,0x0d,0x0a,0xd7,0x99,0xd7,0xa9,0x0d,0x0a,0xd7,0x9b,0xd7,0x9a,
+
0x0d,0x0a,0xd7,0x90,0xd7,0xa0,0xd7,0x99,0x0d,0x0a,0xd7,0x94,0xd7,0x9d,0x0d,0x0a,
+
0xd7,0x93,0xd7,0x95,0xd7,0x9c,0xd7,0xa8,0x0d,0x0a,0xd7,0x90,0xd7,0x9e,0xd7,0xa8,
+
0x0d,0x0a,0xd7,0xa2,0xd7,0x93,0x0d,0x0a,0xd7,0x9c,0xd7,0x90,0xd7,0x97,0xd7,0xa8,
+
0x0d,0x0a,0xd7,0x99,0xd7,0xa9,0xd7,0xa8,0xd7,0x90,0xd7,0x9c,0x0d,0x0a,0xd7,0xa8,
+
0xd7,0xa7,0x0d,0x0a,0xd7,0xa9,0xd7,0xa7,0xd7,0x9c,0x0d,0x0a,0xd7,0x9b,0xd7,0x93,
+
0xd7,0x99,0x0d,0x0a,0xd7,0x9e,0xd7,0x94,0x0d,0x0a,0xd7,0x9c,0xd7,0xa4,0xd7,0xa0,
+
0xd7,0x99,0x0d,0x0a,0xd7,0x90,0xd7,0x97,0xd7,0x93,0x0d,0x0a,0xd7,0x94,0xd7,0x97,
+
0xd7,0x91,0xd7,0xa8,0xd7,0x94,0x0d,0x0a,0xd7,0x9b,0xd7,0x9e,0xd7,0x95,0x0d,0x0a,
+
0xd7,0x96,0xd7,0x90,0xd7,0xaa,0x0d,0x0a,0xd7,0x94,0xd7,0x99,0xd7,0x95,0xd7,0x9d,
+
0x0d,0x0a,0xd7,0x90,0xd7,0x9a,0x0d,0x0a,0xd7,0x9c,0x0d,0x0a,0xd7,0x94,0x0d,0x0a,
+
0xd7,0x9b,0x0d,0x0a,0xd7,0x90,0xd7,0x99,0xd7,0x9f,0x0d,0x0a,0xd7,0x90,0xd7,0xaa,
+
0xd7,0x9e,0xd7,0x95,0xd7,0x9c,0x0d,0x0a,0xd7,0xa9,0xd7,0x9c,0xd7,0x90,0x0d,0x0a,
+
0xd7,0x9b,0xd7,0x91,0xd7,0xa8,0x0d,0x0a,0xd7,0xa2,0xd7,0x95,0xd7,0x93,0x0d,0x0a,
+
0xd7,0x9c,0xd7,0x95,0x0d,0x0a,0xd7,0x96,0xd7,0x95,0x0d,0x0a,0xd7,0x90,0xd7,0x9c,
+
0x0d,0x0a,0xd7,0x91,0xd7,0x9f,0x0d,0x0a,0xd7,0x90,0xd7,0x95,0xd7,0xaa,0xd7,0x95,
+
0x0d,0x0a,0xd7,0xa9,0xd7,0xa0,0xd7,0x99,0x0d,0x0a,0xd7,0x91,0xd7,0x99,0xd7,0xaa,
+
0x0d,0x0a,0xd7,0x99,0xd7,0x93,0xd7,0x99,0x0d,0x0a,0xd7,0x9b,0xd7,0x9e,0xd7,0x94,
+
0x0d,0x0a,0xd7,0x91,0xd7,0x99,0xd7,0x95,0xd7,0xaa,0xd7,0xa8,0x0d,0x0a,0xd7,0x95,
+
0xd7,0x9c,0xd7,0x90,0x0d,0x0a,0xd7,0x94,0xd7,0x9e,0xd7,0x9e,0xd7,0xa9,0xd7,0x9c,
+
0xd7,0x94,0x0d,0x0a,0xd7,0x90,0xd7,0x97,0xd7,0xa8,0xd7,0x99,0x0d,0x0a,0xd7,0x97,
+
0xd7,0x91,0xd7,0xa8,0xd7,0xaa,0x0d,0x0a,0xd7,0x94,0xd7,0x99,0xd7,0xaa,0xd7,0x94,
+
0x0d,0x0a,0xd7,0xa9,0xd7,0x9c,0xd7,0x95,0x0d,0x0a,0xd7,0x94,0xd7,0x99,0xd7,0x95,
+
0x0d,0x0a,0xd7,0xa0,0xd7,0x92,0xd7,0x93,0x0d,0x0a,0xd7,0x91,0xd7,0x9b,0xd7,0x9c,
+
0x0d,0x0a,0xd7,0x90,0xd7,0x91,0xd7,0x99,0xd7,0x91,0x0d,0x0a,0xd7,0xa8,0xd7,0x90,
+
0xd7,0xa9,0x0d,0x0a,0xd7,0x91,0xd7,0x99,0xd7,0xa9,0xd7,0xa8,0xd7,0x90,0xd7,0x9c,
+
0x0d,0x0a,0xd7,0x9c,0xd7,0x99,0x0d,0x0a,0xd7,0xa9,0xd7,0xa0,0xd7,0x99,0xd7,0x9d,
+
0x0d,0x0a,0xd7,0xa4,0xd7,0x99,0x0d,0x0a,0xd7,0x91,0xd7,0x95,0x0d,0x0a,0xd7,0x9e,
+
0x0d,0x0a,0xd7,0x9e,0xd7,0x90,0xd7,0x95,0xd7,0x93,0x0d,0x0a,0xd7,0x9c,0xd7,0x94,
+
0xd7,0x99,0xd7,0x95,0xd7,0xaa,0x0d,0x0a,0xd7,0xa9,0xd7,0x94,0xd7,0x95,0xd7,0x90,
+
0x0d,0x0a,0xd7,0x9e,0xd7,0x99,0x0d,0x0a,0xd7,0x90,0xd7,0x9c,0xd7,0xa3,0x0d,0x0a,
+
0xd7,0x90,0xd7,0x9c,0xd7,0x90,0x0d,0x0a,0xd7,0x90,0xd7,0xa3,0x0d,0x0a,0xd7,0x90,
+
0xd7,0x97,0xd7,0xa8,0x0d,0x0a,0xd7,0x94,0xd7,0x96,0xd7,0x94,0x0d,0x0a,0xd7,0x90,
+
0xd7,0x97,0xd7,0xaa,0x0d,0x0a,0xd7,0x91,0xd7,0x91,0xd7,0x99,0xd7,0xaa,0x0d,0x0a,
+
0xd7,0x90,0xd7,0x9c,0xd7,0x94,0x0d,0x0a,0xd7,0x90,0xd7,0xa0,0xd7,0x97,0xd7,0xa0,
+ 0xd7,0x95,0x0d,0x0a
+ };
+
+static std::map<std::string, std::pair<const unsigned char*, unsigned int> >
createResourceMap() {
+ std::map<std::string, std::pair<const unsigned char*, unsigned int> > m;
+ m["stopwords/en"] = std::pair <const unsigned char*, unsigned
int>(stopwords_en, sizeof stopwords_en);
+ m["stopwords/fra"] = std::pair <const unsigned char*, unsigned
int>(stopwords_fra, sizeof stopwords_fra);
+ m["stopwords/he"] = std::pair <const unsigned char*, unsigned
int>(stopwords_he, sizeof stopwords_he);
+ return m;
+}
+
+static std::map<std::string, std::pair<const unsigned char*, unsigned int> >
resourceMap = createResourceMap();
+
+std::string getResourceAsString(const std::string &name) {
+ std::map<std::string, std::pair<const unsigned char*, unsigned int>
>::iterator it = resourceMap.find(name);
+ if (it != resourceMap.end()) {
+ return std::string((const char*)resourceMap[name].first,
resourceMap[name].second);
+ }
+ return "";
+}
diff --git a/zimwriterfs/resourceTools.h b/zimwriterfs/resourceTools.h
new file mode 100644
index 0000000..c040f86
--- /dev/null
+++ b/zimwriterfs/resourceTools.h
@@ -0,0 +1,10 @@
+#ifndef OPENZIM_ZIMWRITERFS_RESOURCETOOLS_H
+#define OPENZIM_ZIMWRITERFS_RESOURCETOOLS_H
+
+#include <string>
+
+std::string getResourceAsString(const std::string &name);
+
+
+#endif // OPENZIM_ZIMWRITERFS_RESOURCETOOLS_H
+
diff --git a/zimwriterfs/zimwriterfs.cpp b/zimwriterfs/zimwriterfs.cpp
index 1826c31..52ed7ea 100644
--- a/zimwriterfs/zimwriterfs.cpp
+++ b/zimwriterfs/zimwriterfs.cpp
@@ -58,6 +58,7 @@
pthread_mutex_t verboseMutex;
bool inflateHtmlFlag = false;
bool uniqueNamespace = false;
+bool createFullTextIndex = false;
magic_t magic;
@@ -132,6 +133,7 @@
std::cout << "\t-x, --inflateHtml\ttry to inflate HTML files before packing
(*.html, *.htm, ...)" << std::endl;
std::cout << "\t-u, --uniqueNamespace\tput everything in the same namespace
'A'. Might be necessary to avoid problems with dynamic/javascript data
loading." << std::endl;
std::cout << "\t-r, --redirects\t\tpath to the CSV file with the list of
redirects (url, title, target_url tab separated)." << std::endl;
+ std::cout << "\t-i, --createFullTextIndex\t\tIndex the content and add it to
the ZIM." << std::endl;
std::cout << std::endl;
std::cout << "Example:" << std::endl;
@@ -228,7 +230,7 @@
int main(int argc, char** argv) {
ArticleSource source(filenameQueue);
int minChunkSize = 2048;
-
+
/* Argument parsing */
static struct option long_options[] = {
@@ -245,13 +247,14 @@
{"description", required_argument, 0, 'd'},
{"creator", required_argument, 0, 'c'},
{"publisher", required_argument, 0, 'p'},
+ {"createFullTextIndex", no_argument, 0, 'i'},
{0, 0, 0, 0}
};
int option_index = 0;
int c;
do {
- c = getopt_long(argc, argv, "hvxuw:m:f:t:d:c:l:p:r:", long_options,
&option_index);
+ c = getopt_long(argc, argv, "hvixuw:m:f:t:d:c:l:p:r:", long_options,
&option_index);
if (c != -1) {
switch (c) {
@@ -274,6 +277,9 @@
case 'f':
favicon = optarg;
break;
+ case 'i':
+ createFullTextIndex = true;
+ break;
case 'l':
language = optarg;
break;
--
To view, visit https://gerrit.wikimedia.org/r/296912
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I55079339d21d6903634c265f83f4d1c6ba0ac333
Gerrit-PatchSet: 1
Gerrit-Project: openzim
Gerrit-Branch: master
Gerrit-Owner: Mgautierfr <[email protected]>
Gerrit-Reviewer: Kelson <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
