loolwsd/ChildProcessSession.cpp | 2 loolwsd/ChildProcessSession.hpp | 3 loolwsd/Common.hpp | 3 loolwsd/LOOLForKit.cpp | 3 loolwsd/LOOLTool.cpp | 232 ++++++++++++++++++++++++++++++++++++++++ loolwsd/LOOLWSD.cpp | 3 loolwsd/Makefile.am | 4 loolwsd/configure.ac | 2 loolwsd/test/Makefile.am | 8 - scripts/logparse.pl | 9 + scripts/segv-catch.pl | 22 +++ 11 files changed, 275 insertions(+), 16 deletions(-)
New commits: commit 53c60b28421ea65fbc9da9abd4aa7a4a86eb423b Author: Michael Meeks <[email protected]> Date: Thu Apr 14 18:46:32 2016 +0100 Magic to cleanup autotools module linking of unit tests. diff --git a/loolwsd/configure.ac b/loolwsd/configure.ac index e372b25..a5cdd3a 100644 --- a/loolwsd/configure.ac +++ b/loolwsd/configure.ac @@ -4,7 +4,7 @@ AC_PREREQ([2.69]) AC_INIT([loolwsd], [1.6.0], [[email protected]]) -LT_INIT([disable-static]) +LT_INIT([shared, disable-static, dlopen]) AM_INIT_AUTOMAKE([1.11 silent-rules subdir-objects]) diff --git a/loolwsd/test/Makefile.am b/loolwsd/test/Makefile.am index cbeb4cc..d44e787 100644 --- a/loolwsd/test/Makefile.am +++ b/loolwsd/test/Makefile.am @@ -4,10 +4,12 @@ check_PROGRAMS = test AM_CXXFLAGS = $(CPPUNIT_CFLAGS) -check_LTLIBRARIES = \ +noinst_LTLIBRARIES = \ unit-timeout.la unit-prefork.la \ unit-storage.la unit-fonts.la +MAGIC_TO_FORCE_SHLIB_CREATION = -rpath /dummy +AM_LDFLAGS = -module $(MAGIC_TO_FORCE_SHLIB_CREATION) AM_CPPFLAGS = -pthread -I$(top_srcdir) test_CPPFLAGS = -DTDOC=\"$(top_srcdir)/test/data\" @@ -16,13 +18,9 @@ test_LDADD = $(CPPUNIT_LIBS) # unit test modules: unit_fonts_la_SOURCES = UnitFonts.cpp -unit_fonts_la_LDFLAGS = -module unit_timeout_la_SOURCES = UnitTimeout.cpp -unit_timeout_la_LDFLAGS = -module unit_prefork_la_SOURCES = UnitPrefork.cpp -unit_prefork_la_LDFLAGS = -module unit_storage_la_SOURCES = UnitStorage.cpp -unit_storage_la_LDFLAGS = -module if HAVE_LO_PATH SYSTEM_STAMP = @SYSTEMPLATE_PATH@/system_stamp commit 688d1ab6e6bef0ae29ba2617e9f284b5f0dbdb12 Author: Michael Meeks <[email protected]> Date: Thu Apr 14 18:04:19 2016 +0100 Remove confusing ChildProcessSession include on LOOLWSD. Move the ClientPortNumber into both Kit and WSD. diff --git a/loolwsd/ChildProcessSession.cpp b/loolwsd/ChildProcessSession.cpp index a7d00ca..a681221 100644 --- a/loolwsd/ChildProcessSession.cpp +++ b/loolwsd/ChildProcessSession.cpp @@ -42,8 +42,6 @@ using Poco::StringTokenizer; using Poco::Timestamp; using Poco::URI; -int ClientPortNumber = DEFAULT_CLIENT_PORT_NUMBER; - class CallbackNotification: public Notification { public: diff --git a/loolwsd/ChildProcessSession.hpp b/loolwsd/ChildProcessSession.hpp index bb55b68..b7e0af6 100644 --- a/loolwsd/ChildProcessSession.hpp +++ b/loolwsd/ChildProcessSession.hpp @@ -22,9 +22,6 @@ #include "Common.hpp" #include "LOOLSession.hpp" -// The client port number, which is changed via loolwsd args. -extern int ClientPortNumber; - class CallbackWorker; class ChildProcessSession final : public LOOLSession diff --git a/loolwsd/Common.hpp b/loolwsd/Common.hpp index cc9b4d1..2d9db68 100644 --- a/loolwsd/Common.hpp +++ b/loolwsd/Common.hpp @@ -40,6 +40,9 @@ constexpr auto JAILED_DOCUMENT_ROOT = "/user/docs/"; constexpr auto NEW_CHILD_URI = "/loolws/newchild?"; constexpr auto SSL_KEY_FILE = "key.pem"; +// The client port number, both loolwsd and the kits have this. +extern int ClientPortNumber; + #endif /* vim:set shiftwidth=4 softtabstop=4 expandtab: */ diff --git a/loolwsd/LOOLForKit.cpp b/loolwsd/LOOLForKit.cpp index 700273f..256a2c3 100644 --- a/loolwsd/LOOLForKit.cpp +++ b/loolwsd/LOOLForKit.cpp @@ -34,7 +34,6 @@ #include "LOOLKit.hpp" #include "Util.hpp" #include "Unit.hpp" -#include "ChildProcessSession.hpp" #include "security.h" @@ -48,6 +47,8 @@ using Poco::Util::Application; static std::string UnitTestLibrary; static std::atomic<unsigned> ForkCounter( 0 ); +int ClientPortNumber = DEFAULT_CLIENT_PORT_NUMBER; + static int pipeFd = -1; class ChildDispatcher diff --git a/loolwsd/LOOLWSD.cpp b/loolwsd/LOOLWSD.cpp index 38d5935..6c1d790 100644 --- a/loolwsd/LOOLWSD.cpp +++ b/loolwsd/LOOLWSD.cpp @@ -81,7 +81,6 @@ #include "Admin.hpp" #include "Auth.hpp" -#include "ChildProcessSession.hpp" #include "Common.hpp" #include "FileServer.hpp" #include "LOOLProtocol.hpp" @@ -143,6 +142,8 @@ using Poco::XML::InputSource; using Poco::XML::Node; using Poco::XML::NodeList; +int ClientPortNumber = DEFAULT_CLIENT_PORT_NUMBER; + /// New LOK child processes ready to host documents. //TODO: Move to a more sensible namespace. static std::vector<std::shared_ptr<ChildProcess>> newChildren; commit d817fc2c69d848a9f9e1ef756bb0379705576bcb Author: Michael Meeks <[email protected]> Date: Fri Mar 18 21:07:39 2016 +0000 Simple command-line tool for file format conversion. diff --git a/loolwsd/LOOLTool.cpp b/loolwsd/LOOLTool.cpp new file mode 100644 index 0000000..975ed4c --- /dev/null +++ b/loolwsd/LOOLTool.cpp @@ -0,0 +1,232 @@ +/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4; fill-column: 100 -*- */ +/* + * This file is part of the LibreOffice project. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +#include <unistd.h> + +#include <algorithm> +#include <cstdlib> +#include <cstring> +#include <fstream> +#include <iostream> +#include <random> + +#include <Poco/Condition.h> +#include <Poco/Mutex.h> +#include <Poco/Net/HTMLForm.h> +#include <Poco/Net/NetException.h> +#include <Poco/Net/HTTPClientSession.h> +#include <Poco/Net/HTTPRequest.h> +#include <Poco/Net/HTTPResponse.h> +#include <Poco/Net/FilePartSource.h> +#include <Poco/StreamCopier.h> +#include <Poco/URI.h> +#include <Poco/Process.h> +#include <Poco/StringTokenizer.h> +#include <Poco/Thread.h> +#include <Poco/Timespan.h> +#include <Poco/Timestamp.h> +#include <Poco/URI.h> +#include <Poco/Util/Application.h> +#include <Poco/Util/HelpFormatter.h> +#include <Poco/Util/Option.h> +#include <Poco/Util/OptionSet.h> + +#include "Common.hpp" +#include "LOOLProtocol.hpp" +#include "Util.hpp" + +#include <Poco/Util/Application.h> +#include <Poco/Util/OptionSet.h> + +class Tool: public Poco::Util::Application +{ +public: + Tool(); + ~Tool() {} + + unsigned _numWorkers; + std::string _serverURI; + std::string _destinationFormat; + std::string _destinationDir; // FIXME: implement me. + +protected: + void defineOptions(Poco::Util::OptionSet& options) override; + void handleOption(const std::string& name, const std::string& value) override; + int main(const std::vector<std::string>& args) override; +}; + + +using namespace LOOLProtocol; + +using Poco::Condition; +using Poco::Mutex; +using Poco::Net::HTTPClientSession; +using Poco::Net::HTTPRequest; +using Poco::Net::HTTPResponse; +using Poco::Runnable; +using Poco::StringTokenizer; +using Poco::Thread; +using Poco::Timespan; +using Poco::Timestamp; +using Poco::URI; +using Poco::Util::Application; +using Poco::Util::HelpFormatter; +using Poco::Util::Option; +using Poco::Util::OptionSet; + +class Worker: public Runnable +{ +public: + Tool& _app; + std::vector< std::string > _files; + Worker(Tool& app, const std::vector< std::string > & files) : + _app(app), _files(files) + { + } + + void run() override + { + for (auto i : _files) + convertFile(i); + } + + void convertFile(const std::string& document) + { + std::cerr << "convert file " << document << "\n"; + + Poco::URI uri(_app._serverURI); + Poco::Net::HTTPClientSession session(uri.getHost(), uri.getPort()); + Poco::Net::HTTPRequest request(Poco::Net::HTTPRequest::HTTP_POST, "/convert-to"); + + try { + Poco::Net::HTMLForm form; + form.setEncoding(Poco::Net::HTMLForm::ENCODING_MULTIPART); + form.set("format", _app._destinationFormat); + form.addPart("data", new Poco::Net::FilePartSource(document)); + form.prepareSubmit(request); + + // If this results in a Poco::Net::ConnectionRefusedException, loolwsd is not running. + form.write(session.sendRequest(request)); + } + catch (const Poco::Exception &e) + { + std::cerr << "Failed to connect: " << e.name() << + " " << e.message() << "\n"; + return; + } + + Poco::Net::HTTPResponse response; + std::stringstream actualStream; + + // receiveResponse() resulted in a Poco::Net::NoMessageException. + std::istream& responseStream = session.receiveResponse(response); + Poco::StreamCopier::copyStream(responseStream, actualStream); + + // FIXME: implement destinationDir + Poco::Path path(document); + + std::string outPath = path.getBaseName() + "." + _app._destinationFormat; + std::cerr << "write to " << outPath << "\n"; + std::ifstream fileStream(outPath); + std::stringstream expectedStream; + expectedStream << fileStream.rdbuf(); + + // In some cases the result is prefixed with (the UTF-8 encoding of) the Unicode BOM + // (U+FEFF). Skip that. + std::string actualString = actualStream.str(); + if (actualString.size() > 3 && actualString[0] == '\xEF' && actualString[1] == '\xBB' && actualString[2] == '\xBF') + actualString = actualString.substr(3); + } +}; + +Tool::Tool() : + _numWorkers(4), +#if ENABLE_SSL + _serverURI("https://localhost:"; + std::to_string(DEFAULT_CLIENT_PORT_NUMBER) + "/wss"), +#else + _serverURI("http://localhost:"; + std::to_string(DEFAULT_CLIENT_PORT_NUMBER) + "/ws"), +#endif + _destinationFormat("txt") +{ +} + +void Tool::defineOptions(OptionSet& optionSet) +{ + Application::defineOptions(optionSet); + + optionSet.addOption(Option("help", "", "Display help information on command line arguments.") + .required(false).repeatable(false)); + optionSet.addOption(Option("extension", "", "file format extension to convert to") + .required(false).repeatable(false) + .argument("format")); + optionSet.addOption(Option("outdir", "", "output directory for converted files") + .required(false).repeatable(false)); + optionSet.addOption(Option("parallelism", "", "number of simultaneous threads to use") + .required(false) .repeatable(false) + .argument("threads")); + optionSet.addOption(Option("server", "", "URI of LOOL server") + .required(false).repeatable(false) + .argument("uri")); +} + +void Tool::handleOption(const std::string& optionName, + const std::string& value) +{ + Application::handleOption(optionName, value); + + if (optionName == "help") + { + HelpFormatter helpFormatter(options()); + + helpFormatter.setCommand(commandName()); + helpFormatter.setUsage("OPTIONS"); + helpFormatter.setHeader("LibreOffice On-Line tool."); + helpFormatter.format(std::cout); + std::exit(Application::EXIT_OK); + } + else if (optionName == "format") + _destinationFormat = value; + else if (optionName == "outdir") + _destinationDir = value; + else if (optionName == "threads") + _numWorkers = std::min(std::stoi(value), 1); + else if (optionName == "uri") + _serverURI = value; +} + +int Tool::main(const std::vector<std::string>& args) +{ + Thread *clients[_numWorkers]; + + size_t chunk = (args.size() + _numWorkers - 1) / _numWorkers; + size_t offset = 0; + for (unsigned i = 0; i < _numWorkers; i++) + { + clients[i] = new Thread(); + size_t toCopy = std::min(args.size() - offset, chunk); + if (toCopy > 0) + { + std::vector< std::string > files( toCopy ); + std::copy( args.begin() + offset, args.begin() + offset + chunk, files.begin() ); + offset += toCopy; + clients[i]->start(*(new Worker(*this, files))); + } + } + + for (unsigned i = 0; i < _numWorkers; i++) + { + clients[i]->join(); + } + + return Application::EXIT_OK; +} + +POCO_APP_MAIN(Tool) + +/* vim:set shiftwidth=4 softtabstop=4 expandtab: */ diff --git a/loolwsd/Makefile.am b/loolwsd/Makefile.am index 0748c6c..d5feb3a 100644 --- a/loolwsd/Makefile.am +++ b/loolwsd/Makefile.am @@ -1,6 +1,6 @@ SUBDIRS = . test -bin_PROGRAMS = loolwsd loolforkit loolmap loolmount +bin_PROGRAMS = loolwsd loolforkit loolmap loolmount looltool dist_bin_SCRIPTS = loolwsd-systemplate-setup @@ -69,6 +69,8 @@ loolmount_SOURCES = loolmount.c loolmap_SOURCES = loolmap.c +looltool_SOURCES = LOOLTool.cpp + noinst_HEADERS = Admin.hpp \ AdminModel.hpp \ Auth.hpp \ commit 349e2a5a49eb4ebb0d5e542bf07a4aaf7a3aa1b6 Author: Michael Meeks <[email protected]> Date: Thu Apr 14 14:30:27 2016 +0100 Add SEGV and trace hunter, and update log parser a little. diff --git a/scripts/logparse.pl b/scripts/logparse.pl index 72676ac..75444ef 100755 --- a/scripts/logparse.pl +++ b/scripts/logparse.pl @@ -98,7 +98,12 @@ while (my $line = shift @input) { } } - if ($line =~/Initializing wsd/) { + # systemd[1]: loolwsd.service: main process exited, code=killed, status=11/SEGV + if ($line =~m/loolwsd.service: main process exited.*status=(.*)$/) { + print "loolwsd exit: $1\n"; + } + + if ($line =~m/Initializing wsd/) { print "Re-started\n"; clear_state(\%sessions, \%lok_starting, \%lok_running); } @@ -109,7 +114,7 @@ while (my $line = shift @input) { # [loolkit ] loolkit [1689] is ready. # [loolbroker ] Child 1536 terminated. # [loolbroker ] Child process [1689] exited with code: 0. - if ($line =~ m/loolbroker.*Spawned kit \[(\d+)\]./) { + if ($line =~ m/loolbroker.*Forked kit \[(\d+)\]./) { my $pid = $1; $lok_starting{$pid} = 1; $pevent = "newkit\t\"$pid\""; diff --git a/scripts/segv-catch.pl b/scripts/segv-catch.pl new file mode 100755 index 0000000..dc8bd8e --- /dev/null +++ b/scripts/segv-catch.pl @@ -0,0 +1,22 @@ +#!/usr/bin/perl -w + +use strict; + +while (<STDIN>) { + my $line = $_; + $line =~ s/\r*\n*//g; + + $line =~ m/Fatal signal received: (.*)$/ || next; + my $signal = $1; + + # 2016-04-12T20:06:49.626770+00:00 ip-172-31-34-231 loolwsd[5918]: kit-02721 Fatal signal received: SIGSEGV + + $line =~ m/(\d+-\d+-\d+)T(\d+:\d+:\d+\.)\S+\s+\S+\s+(\S+)\s+(.*)$/ || die "bad line: '$line'"; + my ($day, $time, $process, $msg) = ($1, $2, $3, $4); + + print "$day $time $process $signal\n"; +} +# 2016-04-12T20:06:49.627111+00:00 ip-172-31-34-231 loolwsd[5918]: Backtrace: +# 2016-04-12T20:06:49.627455+00:00 ip-172-31-34-231 loolwsd[5918]: /usr/bin/loolforkit() [0x43e86d] +# 2016-04-12T20:06:49.627795+00:00 ip-172-31-34-231 loolwsd[5918]: /lib64/libpthread.so.0(+0xf890) [0x7f9389f97890] + _______________________________________________ Libreoffice-commits mailing list [email protected] https://lists.freedesktop.org/mailman/listinfo/libreoffice-commits
