Petr Onderka has uploaded a new change for review.
https://gerrit.wikimedia.org/r/84748
Change subject: Handle progress reporting even when tellg() doesn't work
......................................................................
Handle progress reporting even when tellg() doesn't work
Change-Id: Ib57e9694a3656a161cf33f3b730981668fb462da
---
M XmlInput/WrapperInputStream.h
M main.cpp
2 files changed, 20 insertions(+), 11 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/operations/dumps/incremental
refs/changes/48/84748/1
diff --git a/XmlInput/WrapperInputStream.h b/XmlInput/WrapperInputStream.h
index 2cecc3e..ea11593 100644
--- a/XmlInput/WrapperInputStream.h
+++ b/XmlInput/WrapperInputStream.h
@@ -6,18 +6,24 @@
{
private:
std::istream &wrapped;
- std::function<void()> sideAction;
+ std::function<void(int)> sideAction;
public:
- WrapperInputStream(std::istream &wrapped, std::function<void()> sideAction
= nullptr)
+ WrapperInputStream(std::istream &wrapped, std::function<void()> sideAction)
+ : wrapped(wrapped), sideAction([=](int){ sideAction(); })
+ {}
+
+ WrapperInputStream(std::istream &wrapped, std::function<void(int)>
sideAction = nullptr)
: wrapped(wrapped), sideAction(sideAction)
{}
virtual int read(XML_Char *buf, size_t bufLen) OVERRIDE
{
- if (sideAction != nullptr)
- sideAction();
-
wrapped.read(buf, bufLen);
- return wrapped.gcount();
+ auto count = wrapped.gcount();
+
+ if (sideAction != nullptr)
+ sideAction(count);
+
+ return count;
}
};
\ No newline at end of file
diff --git a/main.cpp b/main.cpp
index affbd95..631c785 100644
--- a/main.cpp
+++ b/main.cpp
@@ -149,10 +149,13 @@
void createDumpCore(ProgressWriterWrapper& writer, std::istream& inputStream)
{
std::uint64_t i = 0;
- std::function<void ()> offsetReportingFunction = [&]()
+ std::uint64_t totalRead = 0;
+ std::function<void(int)> offsetReportingFunction = [&](int read)
{
+ totalRead += read;
+
if (i % 100 == 0)
- writer.ReportOffset(inputStream.tellg());
+ writer.ReportOffset(totalRead);
i++;
};
@@ -223,8 +226,7 @@
dumpBackupProcess.set_buffer_limit(exec_stream_t::s_out, 8192);
dumpBackupProcess.start(phpPath, dumpBackupParameters + " --full --stub");
- WrapperInputStream dumpBackupStream(dumpBackupProcess.out(),
- [&]()
+ std::function<void()> progressForwardingFunction = [&]()
{
auto &stream = dumpBackupProcess.err();
char buffer[1024];
@@ -234,7 +236,8 @@
count = stream.readsome(buffer, 1024);
std::cerr.write(buffer, count);
} while (count != 0);
- });
+ };
+ WrapperInputStream dumpBackupStream(dumpBackupProcess.out(),
progressForwardingFunction);
XmlMediawikiProcessor::Process(&writer, dumpBackupStream);
--
To view, visit https://gerrit.wikimedia.org/r/84748
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: Ib57e9694a3656a161cf33f3b730981668fb462da
Gerrit-PatchSet: 1
Gerrit-Project: operations/dumps/incremental
Gerrit-Branch: gsoc
Gerrit-Owner: Petr Onderka <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
