common/Util.hpp | 23 ++++++++ net/Socket.cpp | 126 +++++++++++++++++++++++++++++++++++++++++++++-- net/Socket.hpp | 22 +++++++- net/WebSocketHandler.hpp | 8 +- wsd/LOOLWSD.cpp | 24 +++++--- 5 files changed, 184 insertions(+), 19 deletions(-)
New commits: commit b726c6c4392df213a4da0a1a8b4ba58654a65e8b Author: Michael Meeks <michael.me...@collabora.com> AuthorDate: Wed May 22 11:25:28 2019 +0100 Commit: Andras Timar <andras.ti...@collabora.com> CommitDate: Thu Jun 13 10:15:23 2019 +0200 Initial chunked transfer encoding. Important for convert-to on larger documents and/or with newer curls. Change-Id: Id18be6d22741a3af7cee39a069c509e4f662977b Reviewed-on: https://gerrit.libreoffice.org/72748 Reviewed-by: Andras Timar <andras.ti...@collabora.com> Tested-by: Andras Timar <andras.ti...@collabora.com> diff --git a/common/Util.hpp b/common/Util.hpp index 07af2a6a4..0190f953b 100644 --- a/common/Util.hpp +++ b/common/Util.hpp @@ -162,6 +162,18 @@ namespace Util // Extract all json entries into a map. std::map<std::string, std::string> JsonToMap(const std::string& jsonString); + inline int hexDigitFromChar(char c) + { + if (c >= '0' && c <= '9') + return c - '0'; + else if (c >= 'a' && c <= 'f') + return c - 'a' + 10; + else if (c >= 'A' && c <= 'F') + return c - 'A' + 10; + else + return -1; + } + /// Dump a lineof data as hex inline std::string stringifyHexLine( const std::vector<char> &buffer, @@ -230,6 +242,17 @@ namespace Util os.flush(); } + inline std::string dumpHex (const char *legend, const char *prefix, + const std::vector<char>::iterator &startIt, + const std::vector<char>::iterator &endIt, + bool skipDup = true, const unsigned int width = 32) + { + std::ostringstream oss; + std::vector<char> data(startIt, endIt); + dumpHex(oss, legend, prefix, data, skipDup, width); + return oss.str(); + } + /// Trim spaces from the left. Just spaces. inline std::string& ltrim(std::string& s) { diff --git a/net/Socket.cpp b/net/Socket.cpp index 2737c6f1c..5b863c8ae 100644 --- a/net/Socket.cpp +++ b/net/Socket.cpp @@ -432,14 +432,21 @@ bool ServerSocket::bind(Type type, int port) #ifndef MOBILEAPP +// For a verbose life, tweak here: +#if 0 +# define LOG_CHUNK(X) LOG_TRC(X) +#else +# define LOG_CHUNK(X) +#endif + bool StreamSocket::parseHeader(const char *clientName, Poco::MemoryInputStream &message, Poco::Net::HTTPRequest &request, - size_t *requestSize) + MessageMap *map) { LOG_TRC("#" << getFD() << " handling incoming " << _inBuffer.size() << " bytes."); - assert(!requestSize || *requestSize == 0); + assert(!map || (map->_headerSize == 0 && map->_messageSize == 0)); // Find the end of the header, if any. static const std::string marker("\r\n\r\n"); @@ -453,8 +460,11 @@ bool StreamSocket::parseHeader(const char *clientName, // Skip the marker. itBody += marker.size(); - if (requestSize) - *requestSize = static_cast<size_t>(itBody - _inBuffer.begin()); + if (map) // a reasonable guess so far + { + map->_headerSize = static_cast<size_t>(itBody - _inBuffer.begin()); + map->_messageSize = map->_headerSize; + } try { @@ -485,6 +495,8 @@ bool StreamSocket::parseHeader(const char *clientName, LOG_DBG("Not enough content yet: ContentLength: " << contentLength << ", available: " << available); return false; } + if (map) + map->_messageSize += contentLength; if (request.getExpectContinue() && !_sentHTTPContinue) { @@ -494,6 +506,79 @@ bool StreamSocket::parseHeader(const char *clientName, sizeof("HTTP/1.1 100 Continue\r\n\r\n") - 1); _sentHTTPContinue = true; } + + if (request.getChunkedTransferEncoding()) + { + // keep the header + if (map) + map->_spans.push_back(std::pair<size_t, size_t>(0, itBody - _inBuffer.begin())); + + int chunk = 0; + while (itBody != _inBuffer.end()) + { + auto chunkStart = itBody; + + // skip whitespace + for (; itBody != _inBuffer.end() && isascii(*itBody) && isspace(*itBody); ++itBody) + ; // skip. + + // each chunk is preceeded by its length in hex. + size_t chunkLen = 0; + for (; itBody != _inBuffer.end(); ++itBody) + { + int digit = Util::hexDigitFromChar(*itBody); + if (digit >= 0) + chunkLen = chunkLen * 16 + digit; + else + break; + } + + LOG_CHUNK("Chunk of length " << chunkLen); + + for (; itBody != _inBuffer.end() && *itBody != '\n'; ++itBody) + ; // skip to end of line + + if (itBody != _inBuffer.end()) + itBody++; /* \n */; + + // skip the chunk. + auto chunkOffset = itBody - _inBuffer.begin(); + auto chunkAvailable = _inBuffer.size() - chunkOffset; + + if (chunkLen == 0) // we're complete. + { + map->_messageSize = chunkOffset; + return true; + } + + if (chunkLen > chunkAvailable + 2) + { + LOG_DBG("Not enough content yet in chunk " << chunk << + " starting at offset " << (chunkStart - _inBuffer.begin()) << + " chunk len: " << chunkLen << ", available: " << chunkAvailable); + return false; + } + itBody += chunkLen; + + map->_spans.push_back(std::pair<size_t,size_t>(chunkOffset, chunkLen)); + + if (*itBody != '\r' || *(itBody + 1) != '\n') + { + LOG_ERR("Missing \\r\\n at end of chunk " << chunk << " of length " << chunkLen); + LOG_CHUNK("Chunk " << chunk << " is: \n" << Util::dumpHex("", "", chunkStart, itBody + 1, false)); + return false; // TODO: throw something sensible in this case + } + else + { + LOG_CHUNK("Chunk " << chunk << " is: \n" << Util::dumpHex("", "", chunkStart, itBody + 1, false)); + } + + itBody+=2; + chunk++; + } + LOG_TRC("Not enough chunks yet, so far " << chunk << " chunks of total length " << (itBody - _inBuffer.begin())); + return false; + } } catch (const Poco::Exception& exc) { @@ -513,6 +598,39 @@ bool StreamSocket::parseHeader(const char *clientName, return true; } +bool StreamSocket::compactChunks(MessageMap *map) +{ + assert (map); + if (!map->_spans.size()) + return false; // single message. + + LOG_CHUNK("Pre-compact " << map->_spans.size() << " chunks: \n" << + Util::dumpHex("", "", _inBuffer.begin(), _inBuffer.end(), false)); + + char *first = &_inBuffer[0]; + char *dest = first; + for (auto &span : map->_spans) + { + std::memmove(dest, &_inBuffer[span.first], span.second); + dest += span.second; + } + + // Erase the duplicate bits. + size_t newEnd = dest - first; + size_t gap = map->_messageSize - newEnd; + _inBuffer.erase(_inBuffer.begin() + newEnd, _inBuffer.begin() + map->_messageSize); + + LOG_CHUNK("Post-compact with erase of " << newEnd << " to " << map->_messageSize << " giving: \n" << + Util::dumpHex("", "", _inBuffer.begin(), _inBuffer.end(), false)); + + // shrink our size to fit + map->_messageSize -= gap; + + dumpState(std::cerr); + + return true; +} + namespace HttpHelper { void sendUncompressedFileContent(const std::shared_ptr<StreamSocket>& socket, diff --git a/net/Socket.hpp b/net/Socket.hpp index de20991e1..49cfeab49 100644 --- a/net/Socket.hpp +++ b/net/Socket.hpp @@ -930,9 +930,21 @@ public: return socket; } + /// Messages can be in chunks, only parts of message being valid. + struct MessageMap { + MessageMap() : _headerSize(0), _messageSize(0) {} + /// Size of HTTP headers + size_t _headerSize; + /// Entire size of data associated with this message + size_t _messageSize; + // offset + lengths to collate into the real stream + std::vector<std::pair<size_t, size_t>> _spans; + }; + /// Remove the first @count bytes from input buffer - void eraseFirstInputBytes(size_t count) + void eraseFirstInputBytes(const MessageMap &map) { + size_t count = map._headerSize; size_t toErase = std::min(count, _inBuffer.size()); if (toErase < count) LOG_ERR("#" << getFD() << ": attempted to remove: " << count << " which is > size: " << _inBuffer.size() << " clamped to " << toErase); @@ -940,12 +952,16 @@ public: _inBuffer.erase(_inBuffer.begin(), _inBuffer.begin() + count); } + /// Compacts chunk headers away leaving just the data we want + /// returns true if we did any re-sizing/movement of _inBuffer. + bool compactChunks(MessageMap *map); + /// Detects if we have an HTTP header in the provided message and /// populates a request for that. bool parseHeader(const char *clientLoggingName, Poco::MemoryInputStream &message, Poco::Net::HTTPRequest &request, - size_t *requestSize = nullptr); + MessageMap *map = nullptr); /// Get input/output statistics on this stream void getIOStats(uint64_t &sent, uint64_t &recv) @@ -966,6 +982,8 @@ public: protected: + std::vector<std::pair<size_t, size_t>> findChunks(Poco::Net::HTTPRequest &request); + /// Called when a polling event is received. /// @events is the mask of events that triggered the wake. void handlePoll(SocketDisposition &disposition, diff --git a/net/WebSocketHandler.hpp b/net/WebSocketHandler.hpp index 472360ab6..f483661be 100644 --- a/net/WebSocketHandler.hpp +++ b/net/WebSocketHandler.hpp @@ -683,7 +683,7 @@ protected: LOG_TRC("Incoming client websocket upgrade response: " << std::string(&socket->getInBuffer()[0], socket->getInBuffer().size())); bool bOk = false; - size_t responseSize = 0; + StreamSocket::MessageMap map; try { @@ -699,7 +699,7 @@ protected: marker.begin(), marker.end()); if (itBody != socket->getInBuffer().end()) - responseSize = itBody - socket->getInBuffer().begin() + marker.size(); + map._headerSize = itBody - socket->getInBuffer().begin() + marker.size(); } if (response.getStatus() == Poco::Net::HTTPResponse::HTTP_SWITCHING_PROTOCOLS && @@ -728,7 +728,7 @@ protected: LOG_DBG("handleClientUpgrade exception caught."); } - if (!bOk || responseSize == 0) + if (!bOk || map._headerSize == 0) { LOG_ERR("Bad websocker server response."); @@ -737,7 +737,7 @@ protected: } setWebSocket(); - socket->eraseFirstInputBytes(responseSize); + socket->eraseFirstInputBytes(map); } #endif diff --git a/wsd/LOOLWSD.cpp b/wsd/LOOLWSD.cpp index 90659923a..89171b339 100644 --- a/wsd/LOOLWSD.cpp +++ b/wsd/LOOLWSD.cpp @@ -1826,9 +1826,7 @@ private: try { #ifndef MOBILEAPP - size_t requestSize = 0; - - if (!socket->parseHeader("Prisoner", message, request, &requestSize)) + if (!socket->parseHeader("Prisoner", message, request)) return; LOG_TRC("Child connection with URI [" << LOOLWSD::anonymizeUrl(request.getURI()) << "]."); @@ -2048,16 +2046,23 @@ private: return; } - Poco::MemoryInputStream message(&socket->getInBuffer()[0], - socket->getInBuffer().size());; + Poco::MemoryInputStream startmessage(&socket->getInBuffer()[0], + socket->getInBuffer().size());; Poco::Net::HTTPRequest request; - size_t requestSize = 0; - if (!socket->parseHeader("Client", message, request, &requestSize)) + StreamSocket::MessageMap map; + if (!socket->parseHeader("Client", startmessage, request, &map)) return; try { + // We may need to re-write the chunks moving the inBuffer. + socket->compactChunks(&map); + Poco::MemoryInputStream message(&socket->getInBuffer()[0], + socket->getInBuffer().size()); + // update the read cursor - headers are not altered by chunks. + message.seekg(startmessage.tellg(), std::ios::beg); + // Check and remove the ServiceRoot from the request.getURI() if (!Util::startsWith(request.getURI(), LOOLWSD::ServiceRoot)) throw BadRequestException("The request does not start with prefix: " + LOOLWSD::ServiceRoot); @@ -2166,7 +2171,7 @@ private: // if we succeeded - remove the request from our input buffer // we expect one request per socket - socket->eraseFirstInputBytes(requestSize); + socket->eraseFirstInputBytes(map); #else Poco::Net::HTTPRequest request; handleClientWsUpgrade(request, std::string(socket->getInBuffer().data(), socket->getInBuffer().size()), disposition); @@ -2331,7 +2336,8 @@ private: return "application/octet-stream"; } - void handlePostRequest(const Poco::Net::HTTPRequest& request, Poco::MemoryInputStream& message, + void handlePostRequest(const Poco::Net::HTTPRequest& request, + Poco::MemoryInputStream& message, SocketDisposition &disposition) { LOG_INF("Post request: [" << LOOLWSD::anonymizeUrl(request.getURI()) << "]"); _______________________________________________ Libreoffice-commits mailing list libreoffice-comm...@lists.freedesktop.org https://lists.freedesktop.org/mailman/listinfo/libreoffice-commits