Petr Onderka has submitted this change and it was merged.
Change subject: Handle progress reporting even when tellg() doesn't work
......................................................................
Handle progress reporting even when tellg() doesn't work
Change-Id: Ib57e9694a3656a161cf33f3b730981668fb462da
---
M Dump.cpp
M DumpException.cpp
M DumpException.h
M Objects/IpV4User.cpp
M StringHelpers.cpp
M XmlInput/WrapperInputStream.h
M main.cpp
7 files changed, 69 insertions(+), 30 deletions(-)
Approvals:
Petr Onderka: Verified; Looks good to me, approved
diff --git a/Dump.cpp b/Dump.cpp
index 46d2d31..7afad41 100644
--- a/Dump.cpp
+++ b/Dump.cpp
@@ -34,6 +34,7 @@
if (!stream->is_open())
{
+ errno = 0;
// this feels dangerous, isn't there a better way?
stream = new fstream(fileName, ios::in | ios::out | ios::binary |
ios::trunc);
}
@@ -193,4 +194,4 @@
return std::make_pair(WikitextModel, WikitextFormat);
return modelFormatIndex->Get(id);
-}
\ No newline at end of file
+}
diff --git a/DumpException.cpp b/DumpException.cpp
index 761cc94..f2dfec7 100644
--- a/DumpException.cpp
+++ b/DumpException.cpp
@@ -1,14 +1,22 @@
#include "DumpException.h"
-UserException::UserException(const std::string &message)
+CustomException::CustomException(const std::string& message)
: message(message)
{}
-const char* UserException::what() const NOEXCEPT
+const char* CustomException::what() const NOEXCEPT
{
return message.c_str();
}
-ParametersException::ParametersException(const std::string &message)
+DumpException::DumpException(const std::string& message)
+ : CustomException(message)
+{}
+
+UserException::UserException(const std::string& message)
+ : CustomException(message)
+{}
+
+ParametersException::ParametersException(const std::string& message)
: UserException(message)
{}
diff --git a/DumpException.h b/DumpException.h
index 3c11e6c..b22ef21 100644
--- a/DumpException.h
+++ b/DumpException.h
@@ -4,23 +4,32 @@
#include <string>
#include "common.h"
-class DumpException : public std::exception
-{};
-
-// exception that was most likely caused by user error
-// and should be shown directly to the user
-class UserException : public std::exception
+class CustomException : public std::exception
{
private:
const std::string message;
public:
- UserException(const std::string &message);
+ CustomException(const std::string& message);
virtual const char* what() const NOEXCEPT OVERRIDE;
// GCC 4.6 seems to require this
- virtual ~UserException() NOEXCEPT
+ virtual ~CustomException() NOEXCEPT
{}
+};
+
+class DumpException : public CustomException
+{
+public:
+ DumpException(const std::string& message = std::string());
+};
+
+// exception that was most likely caused by user error
+// and should be shown directly to the user
+class UserException : public CustomException
+{
+public:
+ UserException(const std::string& message);
};
// user exception that was caused by invalid parameters,
@@ -28,5 +37,5 @@
class ParametersException : public UserException
{
public:
- ParametersException(const std::string &message);
+ ParametersException(const std::string& message);
};
diff --git a/Objects/IpV4User.cpp b/Objects/IpV4User.cpp
index 54201ec..a426aae 100644
--- a/Objects/IpV4User.cpp
+++ b/Objects/IpV4User.cpp
@@ -19,9 +19,10 @@
for (int i = 0; i < 4; i++)
{
- bool success;
- long intPart = tryParseLong(stringParts.at(i), success);
- if (!success || intPart > 255 || intPart < 0)
+ bool parseSuccess;
+ long intPart = tryParseLong(stringParts.at(i), parseSuccess);
+
+ if (!parseSuccess || intPart > 255 || intPart < 0)
return 0;
result |= (uint32_t)(uint8_t)intPart << (8 * i);
@@ -82,4 +83,4 @@
RevisionFlags IpV4User::UserKind() const
{
return RevisionFlags::IpV4User;
-}
\ No newline at end of file
+}
diff --git a/StringHelpers.cpp b/StringHelpers.cpp
index 3995c38..5e1cbe0 100644
--- a/StringHelpers.cpp
+++ b/StringHelpers.cpp
@@ -1,4 +1,6 @@
#include "StringHelpers.h"
+#include "DumpException.h"
+#include "format.h"
#include <sstream>
// http://stackoverflow.com/a/236803/41071
@@ -20,6 +22,12 @@
long tryParseLong(const std::string &s, bool &success, int radix)
{
+ if (errno != 0)
+ {
+ throw DumpException(
+ str(fmt::Format("errno was already set ({0}: {1})") << errno <<
strerror(errno)));
+ }
+
char* end;
const char* start = s.c_str();
long result = strtol(start, &end, radix);
@@ -31,7 +39,8 @@
}
else
{
+ errno = 0;
success = false;
return 0;
}
-}
\ No newline at end of file
+}
diff --git a/XmlInput/WrapperInputStream.h b/XmlInput/WrapperInputStream.h
index 2cecc3e..ea11593 100644
--- a/XmlInput/WrapperInputStream.h
+++ b/XmlInput/WrapperInputStream.h
@@ -6,18 +6,24 @@
{
private:
std::istream &wrapped;
- std::function<void()> sideAction;
+ std::function<void(int)> sideAction;
public:
- WrapperInputStream(std::istream &wrapped, std::function<void()> sideAction
= nullptr)
+ WrapperInputStream(std::istream &wrapped, std::function<void()> sideAction)
+ : wrapped(wrapped), sideAction([=](int){ sideAction(); })
+ {}
+
+ WrapperInputStream(std::istream &wrapped, std::function<void(int)>
sideAction = nullptr)
: wrapped(wrapped), sideAction(sideAction)
{}
virtual int read(XML_Char *buf, size_t bufLen) OVERRIDE
{
- if (sideAction != nullptr)
- sideAction();
-
wrapped.read(buf, bufLen);
- return wrapped.gcount();
+ auto count = wrapped.gcount();
+
+ if (sideAction != nullptr)
+ sideAction(count);
+
+ return count;
}
};
\ No newline at end of file
diff --git a/main.cpp b/main.cpp
index affbd95..62cb24e 100644
--- a/main.cpp
+++ b/main.cpp
@@ -149,10 +149,13 @@
void createDumpCore(ProgressWriterWrapper& writer, std::istream& inputStream)
{
std::uint64_t i = 0;
- std::function<void ()> offsetReportingFunction = [&]()
+ std::uint64_t totalRead = 0;
+ std::function<void(int)> offsetReportingFunction = [&](int read)
{
+ totalRead += read;
+
if (i % 100 == 0)
- writer.ReportOffset(inputStream.tellg());
+ writer.ReportOffset(totalRead);
i++;
};
@@ -185,11 +188,13 @@
if (inputFileName == "-")
{
+ std::cin.exceptions(std::ios::failbit | std::ios::badbit);
createDumpCore(progressWriter, std::cin);
}
else
{
std::ifstream stream(inputFileName, std::ios::binary);
+ stream.exceptions(std::ios::failbit | std::ios::badbit);
createDumpCore(progressWriter, stream);
}
}
@@ -223,8 +228,7 @@
dumpBackupProcess.set_buffer_limit(exec_stream_t::s_out, 8192);
dumpBackupProcess.start(phpPath, dumpBackupParameters + " --full --stub");
- WrapperInputStream dumpBackupStream(dumpBackupProcess.out(),
- [&]()
+ std::function<void()> progressForwardingFunction = [&]()
{
auto &stream = dumpBackupProcess.err();
char buffer[1024];
@@ -234,7 +238,8 @@
count = stream.readsome(buffer, 1024);
std::cerr.write(buffer, count);
} while (count != 0);
- });
+ };
+ WrapperInputStream dumpBackupStream(dumpBackupProcess.out(),
progressForwardingFunction);
XmlMediawikiProcessor::Process(&writer, dumpBackupStream);
@@ -317,4 +322,4 @@
{
std::cerr << "Error: " << ex.what() << "\n";
}
-}
\ No newline at end of file
+}
--
To view, visit https://gerrit.wikimedia.org/r/84748
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: Ib57e9694a3656a161cf33f3b730981668fb462da
Gerrit-PatchSet: 2
Gerrit-Project: operations/dumps/incremental
Gerrit-Branch: gsoc
Gerrit-Owner: Petr Onderka <[email protected]>
Gerrit-Reviewer: Petr Onderka <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
