Mgautierfr has uploaded a new change for review.
https://gerrit.wikimedia.org/r/295519
Change subject: Prepare indexing of articles while creating the zimfile.
......................................................................
Prepare indexing of articles while creating the zimfile.
- Add a makeIndexdb argument to command line.
- Add a customIndexer to the ArticleSource.
Change-Id: I5223e150d402a933d248abb272d442d08f6dcd07
---
M zimwriterfs/article.h
M zimwriterfs/articlesource.cpp
M zimwriterfs/articlesource.h
M zimwriterfs/zimwriterfs.cpp
4 files changed, 98 insertions(+), 9 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/openzim refs/changes/19/295519/1
diff --git a/zimwriterfs/article.h b/zimwriterfs/article.h
index 2585fc6..fffb485 100644
--- a/zimwriterfs/article.h
+++ b/zimwriterfs/article.h
@@ -51,6 +51,8 @@
virtual std::string getMimeType() const;
virtual std::string getRedirectAid() const;
virtual bool shouldCompress() const;
+ virtual bool set_data(std::string inputData) { data = inputData; };
+ virtual std::string get_data() { return data; };
};
class MetadataArticle : public Article {
diff --git a/zimwriterfs/articlesource.cpp b/zimwriterfs/articlesource.cpp
index 6cf5f30..6f9a872 100644
--- a/zimwriterfs/articlesource.cpp
+++ b/zimwriterfs/articlesource.cpp
@@ -19,7 +19,6 @@
*/
#include "articlesource.h"
-#include "article.h"
#include "tools.h"
#include <zim/blob.h>
@@ -43,8 +42,33 @@
unsigned int dataSize = 0;
+typedef std::map<std::string, Article*> ArticlesMap;
+static ArticlesMap articles;
+static pthread_mutex_t articlesMapMutex;
+
+
+static void add_to_articlesMap(const std::string& key, Article* article) {
+ pthread_mutex_lock(&articlesMapMutex);
+ articles[key] = article;
+ pthread_mutex_unlock(&articlesMapMutex);
+}
+
+static Article* pop_from_articlesMap(const std::string& key) {
+ Article* retVal = NULL;
+ pthread_mutex_lock(&articlesMapMutex);
+ std::map<std::string, Article*>::iterator it = articles.find(key);
+ if (it != articles.end()) {
+ retVal = it->second;
+ articles.erase(it);
+ }
+ pthread_mutex_unlock(&articlesMapMutex);
+ return retVal;
+}
+
ArticleSource::ArticleSource(Queue<std::string>& filenameQueue):
- filenameQueue(filenameQueue)
+ filenameQueue(filenameQueue),
+ articleIndexer(NULL),
+ metaIndexArticleDone(false)
{
/* Prepare metadata */
metadataQueue.push("Language");
@@ -72,13 +96,9 @@
return welcome;
}
-Article *article = NULL;
const zim::writer::Article* ArticleSource::getNextArticle() {
std::string path;
-
- if (article != NULL) {
- delete(article);
- }
+ Article *article = NULL;
if (!metadataQueue.empty()) {
path = metadataQueue.front();
@@ -94,6 +114,10 @@
} while (article && article->isInvalid() &&
filenameQueue.popFromQueue(path));
} else {
article = NULL;
+ if ( articleIndexer && !metaIndexArticleDone) {
+ article = articleIndexer->getMetaArticle();
+ metaIndexArticleDone = true;
+ }
}
/* Count mimetypes */
@@ -108,6 +132,10 @@
} else {
counters[mimeType]++;
}
+
+ // Store article for later use (Indexing).
+ // The "later use" is responsible to delete the article.
+ add_to_articlesMap(article->getAid(), article);
}
return article;
@@ -155,6 +183,11 @@
dataSize = value.length();
data = new char[dataSize];
memcpy(data, value.c_str(), dataSize);
+ } else if (aid.substr(0, 3) == "/X/" and articleIndexer) {
+ std::string value = articleIndexer->get_data(aid);
+ dataSize = value.length();
+ data = new char[dataSize];
+ memcpy(data, value.c_str(), dataSize);
} else {
std::string aidPath = directoryPath + "/" + aid;
@@ -182,6 +215,21 @@
dataSize = html.length();
data = new char[dataSize];
memcpy(data, html.c_str(), dataSize);
+
+ Article* article = pop_from_articlesMap(aid);
+ if (!article){
+ std::cerr << "article for " << aid << " no found." << std::endl;
+ std::cerr << "This should not happen. Fix this!!" << std::endl;
+ std::cerr << "Continuing anyway... " << std::endl;
+ } else {
+ if (articleIndexer) {
+ article->set_data(html);
+ // This is to the indexer to correctly delete the article.
+ articleIndexer->indexArticle(article);
+ } else {
+ delete article;
+ }
+ }
} else if (getMimeTypeForFile(aid).find("text/css") == 0) {
std::string css = getFileContent(aidPath);
@@ -251,6 +299,18 @@
}
}
+ // If this is a html, the article is no more in the map
+ // (handle by the indexer or not).
+ // But if not html, article is still in the map.
+ // Remove it and delete it.
+ Article* article = pop_from_articlesMap(aid);
+ if (article) {
+ delete article;
+ }
return zim::Blob(data, dataSize);
}
+void ArticleSource::set_customIndexer(IIndexer* indexer)
+{
+ articleIndexer = indexer;
+}
diff --git a/zimwriterfs/articlesource.h b/zimwriterfs/articlesource.h
index 1ad6524..5d69567 100644
--- a/zimwriterfs/articlesource.h
+++ b/zimwriterfs/articlesource.h
@@ -25,8 +25,26 @@
#include <queue>
#include <fstream>
#include "queue.h"
+#include "article.h"
#include <zim/writer/zimcreator.h>
+
+class IndexMetaArticle : public Article {
+ public:
+ IndexMetaArticle(const std::string& name) {
+ ns = 'X';
+ aid = url = "/X/"+name;
+ title = name;
+ }
+};
+
+class IIndexer
+{
+ public:
+ virtual void indexArticle(Article* article) = 0;
+ virtual IndexMetaArticle* getMetaArticle() = 0;
+ virtual std::string get_data(const std::string& aid) = 0;
+};
class ArticleSource : public zim::writer::ArticleSource {
public:
@@ -34,6 +52,7 @@
virtual const zim::writer::Article* getNextArticle();
virtual zim::Blob getData(const std::string& aid);
virtual std::string getMainPage();
+ virtual void set_customIndexer(IIndexer* indexer);
virtual void init_redirectsQueue_from_file(const std::string& path);
@@ -41,6 +60,8 @@
std::queue<std::string> metadataQueue;
std::queue<std::string> redirectsQueue;
Queue<std::string>& filenameQueue;
+ IIndexer* articleIndexer;
+ bool metaIndexArticleDone;
};
#endif //OPENZIM_ZIMWRITERFS_ARTICLESOURCE_H
diff --git a/zimwriterfs/zimwriterfs.cpp b/zimwriterfs/zimwriterfs.cpp
index 09a62af..e512ea1 100644
--- a/zimwriterfs/zimwriterfs.cpp
+++ b/zimwriterfs/zimwriterfs.cpp
@@ -56,6 +56,7 @@
pthread_mutex_t verboseMutex;
bool inflateHtmlFlag = false;
bool uniqueNamespace = false;
+bool makeIndexdb = false;
magic_t magic;
@@ -130,6 +131,7 @@
std::cout << "\t-x, --inflateHtml\ttry to inflate HTML files before packing
(*.html, *.htm, ...)" << std::endl;
std::cout << "\t-u, --uniqueNamespace\tput everything in the same namespace
'A'. Might be necessary to avoid problems with dynamic/javascript data
loading." << std::endl;
std::cout << "\t-r, --redirects\t\tpath to the CSV file with the list of
redirects (url, title, target_url tab separated)." << std::endl;
+ std::cout << "\t-i, --makeIndexdb\t\tIndex the content and add it to the
ZIM." << std::endl;
std::cout << std::endl;
std::cout << "Example:" << std::endl;
@@ -226,7 +228,7 @@
int main(int argc, char** argv) {
ArticleSource source(filenameQueue);
int minChunkSize = 2048;
-
+
/* Argument parsing */
static struct option long_options[] = {
@@ -243,13 +245,14 @@
{"description", required_argument, 0, 'd'},
{"creator", required_argument, 0, 'c'},
{"publisher", required_argument, 0, 'p'},
+ {"makeIndexdb", no_argument, 0, 'i'},
{0, 0, 0, 0}
};
int option_index = 0;
int c;
do {
- c = getopt_long(argc, argv, "hvxuw:m:f:t:d:c:l:p:r:", long_options,
&option_index);
+ c = getopt_long(argc, argv, "hvixuw:m:f:t:d:c:l:p:r:", long_options,
&option_index);
if (c != -1) {
switch (c) {
@@ -272,6 +275,9 @@
case 'f':
favicon = optarg;
break;
+ case 'i':
+ makeIndexdb = true;
+ break;
case 'l':
language = optarg;
break;
--
To view, visit https://gerrit.wikimedia.org/r/295519
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I5223e150d402a933d248abb272d442d08f6dcd07
Gerrit-PatchSet: 1
Gerrit-Project: openzim
Gerrit-Branch: master
Gerrit-Owner: Mgautierfr <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
