wsd/FileServer.cpp | 163 +++++++++++++++++++++++++++++++++++++++++++++--------
wsd/FileServer.hpp | 12 +++
wsd/LOOLWSD.cpp | 2
3 files changed, 155 insertions(+), 22 deletions(-)
New commits:
commit cda55f3a214eee79bc4c87630cdb63954c64611a
Author: Michael Meeks <michael.me...@collabora.com>
Date: Thu Apr 27 14:14:31 2017 +0100
Various cleanups & improvements to file-serving.
Use what we read at startup as the complete set of files to serve.
Trace log filenames as we read them.
Simplify and accelerate path related checks via the hash.
Kill leak with get_current_dir_name and use the correct path.
diff --git a/wsd/FileServer.cpp b/wsd/FileServer.cpp
index 989168b4..1aa01e11 100644
--- a/wsd/FileServer.cpp
+++ b/wsd/FileServer.cpp
@@ -153,19 +153,15 @@ void FileServerRequestHandler::handleRequest(const
HTTPRequest& request, Poco::M
response.add("Referrer-Policy", "no-referrer");
}
- const auto path = Poco::Path(LOOLWSD::FileServerRoot,
getRequestPathname(request));
- const auto filepath = path.absolute().toString();
- if (filepath.find(LOOLWSD::FileServerRoot) != 0)
- {
- // Accessing unauthorized path.
- throw Poco::FileAccessDeniedException("Invalid or forbidden
file path: [" + filepath + "].");
- }
+ const std::string relPath = getRequestPathname(request);
+ // Is this a file we read at startup - if not; its not for serving.
+ if (FileHash.find(relPath) == FileHash.end())
+ throw Poco::FileAccessDeniedException("Invalid or forbidden
file path: [" + relPath + "].");
+ // Do we have an extension.
const std::size_t extPoint = endPoint.find_last_of('.');
if (extPoint == std::string::npos)
- {
throw Poco::FileNotFoundException("Invalid file.");
- }
const std::string fileType = endPoint.substr(extPoint + 1);
std::string mimeType;
@@ -220,25 +216,22 @@ void FileServerRequestHandler::handleRequest(const
HTTPRequest& request, Poco::M
response.setContentType(mimeType);
response.add("X-Content-Type-Options", "nosniff");
- if(gzip)
+ const std::string *content;
+ if (gzip)
{
response.set("Content-Encoding", "gzip");
- std::ostringstream oss;
- response.write(oss);
- const std::string header = oss.str();
- LOG_TRC("#" << socket->getFD() << ": Sending compressed file
[" << filepath << "]: " << header);
- socket->send(header);
- socket->send(getCompressedFile(filepath));
+ content = getCompressedFile(relPath);
}
else
- {
- std::ostringstream oss;
- response.write(oss);
- const std::string header = oss.str();
- LOG_TRC("#" << socket->getFD() << ": Sending file [" <<
filepath << "]: " << header);
- socket->send(header);
- socket->send(getUncompressedFile(filepath));
- }
+ content = getUncompressedFile(relPath);
+
+ std::ostringstream oss;
+ response.write(oss);
+ const std::string header = oss.str();
+ LOG_TRC("#" << socket->getFD() << ": Sending " <<
+ (!gzip ? "un":"") << "compressed : file [" << relPath <<
"]: " << header);
+ socket->send(header);
+ socket->send(*content);
}
}
catch (const Poco::Net::NotAuthenticatedException& exc)
@@ -283,30 +276,31 @@ void FileServerRequestHandler::handleRequest(const
HTTPRequest& request, Poco::M
}
}
-void FileServerRequestHandler::readDirToHash(std::string path)
+void FileServerRequestHandler::readDirToHash(const std::string &basePath,
const std::string &path)
{
-
struct dirent *currentFile;
struct stat fileStat;
DIR *workingdir;
- workingdir = opendir(path.c_str());
+ LOG_TRC("Pre-reading directory: " << basePath + path << "\n");
+ workingdir = opendir((basePath + path).c_str());
- while((currentFile = readdir(workingdir)) != NULL)
+ while ((currentFile = readdir(workingdir)) != NULL)
{
- if(currentFile->d_name[0] == '.')
+ if (currentFile->d_name[0] == '.')
continue;
- std::string filePath = path + "/" + currentFile->d_name;
- stat(filePath.c_str(), &fileStat);
+ std::string relPath = path + "/" + currentFile->d_name;
+ stat ((basePath + relPath).c_str(), &fileStat);
- if(S_ISDIR(fileStat.st_mode))
- {
- readDirToHash(filePath);
- }
- else if(S_ISREG(fileStat.st_mode))
+ if (S_ISDIR(fileStat.st_mode))
+ readDirToHash(basePath, relPath);
+
+ else if (S_ISREG(fileStat.st_mode))
{
- std::ifstream file(filePath, std::ios::binary);
+ LOG_TRC("Reading file: '" << (basePath + relPath) << " as '" <<
relPath << "'\n");
+
+ std::ifstream file(basePath + relPath, std::ios::binary);
z_stream strm;
strm.zalloc = Z_NULL;
@@ -317,12 +311,12 @@ void FileServerRequestHandler::readDirToHash(std::string
path)
auto buf = std::unique_ptr<char[]>(new char[fileStat.st_size]);
std::string compressedFile = "";
std::string uncompressedFile = "";
- do{
-
+ do {
file.read(&buf[0], fileStat.st_size);
const long unsigned int size = file.gcount();
- if(size == 0)
+ if (size == 0)
break;
+
long unsigned int haveComp;
long unsigned int compSize = compressBound(size);
char *cbuf;
@@ -341,36 +335,36 @@ void FileServerRequestHandler::readDirToHash(std::string
path)
compressedFile = compressedFile + partialcompFile;
uncompressedFile = uncompressedFile + partialuncompFile;
- }while(true);
+ } while(true);
+
std::pair<std::string, std::string> FilePair(uncompressedFile,
compressedFile);
- FileHash.emplace(filePath, FilePair);
+ FileHash.emplace(relPath, FilePair);
}
}
closedir(workingdir);
}
-void FileServerRequestHandler::initializeCompression()
+void FileServerRequestHandler::initialize()
{
- std::string rootPath(get_current_dir_name());
- static const std::vector<std::string> extensionArray
- = {"/loleaflet/dist"};
-
- for(const auto& extension: extensionArray)
+ static const std::vector<std::string> subdirs = { "/loleaflet/dist" };
+ for(const auto& subdir: subdirs)
{
- std::string extensionPath;
- extensionPath = rootPath + extension;
- readDirToHash(extensionPath);
+ try {
+ readDirToHash(LOOLWSD::FileServerRoot, subdir);
+ } catch (...) {
+ LOG_ERR("Failed to read from directory " << subdir);
+ }
}
}
-std::string FileServerRequestHandler::getCompressedFile(std::string path)
+const std::string *FileServerRequestHandler::getCompressedFile(const
std::string &path)
{
- return FileHash[path].second;
+ return &FileHash[path].second;
}
-std::string FileServerRequestHandler::getUncompressedFile(std::string path)
+const std::string *FileServerRequestHandler::getUncompressedFile(const
std::string &path)
{
- return FileHash[path].first;
+ return &FileHash[path].first;
}
std::string FileServerRequestHandler::getRequestPathname(const HTTPRequest&
request)
@@ -381,9 +375,8 @@ std::string
FileServerRequestHandler::getRequestPathname(const HTTPRequest& requ
std::string path(requestUri.getPath());
- // Convert version back to a real file name. Remove first foreslash as the
root ends in one.
- Poco::replaceInPlace(path, std::string("/loleaflet/" LOOLWSD_VERSION_HASH
"/"), std::string("loleaflet/dist/"));
- Poco::replaceInPlace(path, std::string("/loleaflet/"),
std::string("loleaflet/"));
+ // Convert version back to a real file name.
+ Poco::replaceInPlace(path, std::string("/loleaflet/" LOOLWSD_VERSION_HASH
"/"), std::string("/loleaflet/dist/"));
return path;
}
@@ -402,12 +395,13 @@ void FileServerRequestHandler::preprocessFile(const
HTTPRequest& request, Poco::
wopiDomain = Poco::URI(wopiHost).getScheme() + "://" +
Poco::URI(wopiHost).getHost();
}
}
- const auto path = Poco::Path(LOOLWSD::FileServerRoot,
getRequestPathname(request));
- LOG_DBG("Preprocessing file: " << path.toString());
- if (!Poco::File(path).exists())
+ // Is this a file we read at startup - if not; its not for serving.
+ const std::string relPath = getRequestPathname(request);
+ LOG_DBG("Preprocessing file: " << relPath);
+ if (FileHash.find(relPath) == FileHash.end())
{
- LOG_ERR("File [" << path.toString() << "] does not exist.");
+ LOG_ERR("File [" << relPath << "] does not exist.");
// 404 not found
std::ostringstream oss;
@@ -420,10 +414,7 @@ void FileServerRequestHandler::preprocessFile(const
HTTPRequest& request, Poco::
return;
}
- std::string preprocess;
- FileInputStream file(path.toString());
- StreamCopier::copyToString(file, preprocess);
- file.close();
+ std::string preprocess = *getUncompressedFile(relPath);
HTMLForm form(request, message);
const std::string& accessToken = form.get("access_token", "");
@@ -564,7 +555,7 @@ void FileServerRequestHandler::preprocessFile(const
HTTPRequest& request, Poco::
<< preprocess;
socket->send(oss.str());
- LOG_DBG("Sent file: " << path.toString() << ": " << preprocess);
+ LOG_DBG("Sent file: " << relPath << ": " << preprocess);
}
/* vim:set shiftwidth=4 softtabstop=4 expandtab: */
diff --git a/wsd/FileServer.hpp b/wsd/FileServer.hpp
index 54193831..01d5918c 100644
--- a/wsd/FileServer.hpp
+++ b/wsd/FileServer.hpp
@@ -28,13 +28,14 @@ public:
static void handleRequest(const Poco::Net::HTTPRequest& request,
Poco::MemoryInputStream& message, const std::shared_ptr<StreamSocket>& socket);
- static void initializeCompression();
+ /// Read all files that we can serve into memory and compress them.
+ static void initialize();
- static void readDirToHash(std::string path);
+ static void readDirToHash(const std::string &basePath, const std::string
&path);
- static std::string getCompressedFile(std::string path);
+ static const std::string *getCompressedFile(const std::string &path);
- static std::string getUncompressedFile(std::string path);
+ static const std::string *getUncompressedFile(const std::string &path);
private:
static std::map<std::string, std::pair<std::string, std::string>> FileHash;
diff --git a/wsd/LOOLWSD.cpp b/wsd/LOOLWSD.cpp
index 096a9ac9..42e3348f 100644
--- a/wsd/LOOLWSD.cpp
+++ b/wsd/LOOLWSD.cpp
@@ -794,7 +794,7 @@ void LOOLWSD::initialize(Application& self)
LOG_INF("Command trace dumping enabled to file: " << path);
}
- FileServerRequestHandler::initializeCompression();
+ FileServerRequestHandler::initialize();
StorageBase::initialize();
commit be4ebb9b1f24bc0376ead968895c1bbf33f94a05
Author: Aditya Dewan <iit2015...@iiita.ac.in>
Date: Wed Apr 26 19:20:00 2017 +0530
Migrated from deflate to gzip
Change-Id: I64aec9305fe74b6cdcfe15e2f67f9d38cfd6d99a
diff --git a/wsd/FileServer.cpp b/wsd/FileServer.cpp
index 118bf054..989168b4 100644
--- a/wsd/FileServer.cpp
+++ b/wsd/FileServer.cpp
@@ -11,6 +11,10 @@
#include <string>
#include <vector>
+#include <unistd.h>
+#include <sys/stat.h>
+#include <dirent.h>
+#include <zlib.h>
#include <Poco/DateTime.h>
#include <Poco/DateTimeFormat.h>
@@ -45,6 +49,8 @@ using Poco::Net::HTTPBasicCredentials;
using Poco::StreamCopier;
using Poco::Util::Application;
+std::map<std::string, std::pair<std::string, std::string>>
FileServerRequestHandler::FileHash;
+
bool FileServerRequestHandler::isAdminLoggedIn(const HTTPRequest& request,
HTTPResponse &response)
{
@@ -200,8 +206,39 @@ void FileServerRequestHandler::handleRequest(const
HTTPRequest& request, Poco::M
}
}
- bool deflate = request.hasToken("Accept-Encoding", "deflate");
- HttpHelper::sendFile(socket, filepath, mimeType, response,
noCache, deflate);
+ bool gzip = request.hasToken("Accept-Encoding", "gzip");
+
+ response.set("User-Agent", HTTP_AGENT_STRING);
+ response.set("Date",
Poco::DateTimeFormatter::format(Poco::Timestamp(),
Poco::DateTimeFormat::HTTP_FORMAT));
+ if (!noCache)
+ {
+ // 60 * 60 * 24 * 128 (days) = 11059200
+ response.set("Cache-Control", "max-age=11059200");
+ response.set("ETag", "\"" LOOLWSD_VERSION_HASH "\"");
+ }
+
+ response.setContentType(mimeType);
+ response.add("X-Content-Type-Options", "nosniff");
+
+ if(gzip)
+ {
+ response.set("Content-Encoding", "gzip");
+ std::ostringstream oss;
+ response.write(oss);
+ const std::string header = oss.str();
+ LOG_TRC("#" << socket->getFD() << ": Sending compressed file
[" << filepath << "]: " << header);
+ socket->send(header);
+ socket->send(getCompressedFile(filepath));
+ }
+ else
+ {
+ std::ostringstream oss;
+ response.write(oss);
+ const std::string header = oss.str();
+ LOG_TRC("#" << socket->getFD() << ": Sending file [" <<
filepath << "]: " << header);
+ socket->send(header);
+ socket->send(getUncompressedFile(filepath));
+ }
}
}
catch (const Poco::Net::NotAuthenticatedException& exc)
@@ -246,6 +283,96 @@ void FileServerRequestHandler::handleRequest(const
HTTPRequest& request, Poco::M
}
}
+void FileServerRequestHandler::readDirToHash(std::string path)
+{
+
+ struct dirent *currentFile;
+ struct stat fileStat;
+ DIR *workingdir;
+
+ workingdir = opendir(path.c_str());
+
+ while((currentFile = readdir(workingdir)) != NULL)
+ {
+ if(currentFile->d_name[0] == '.')
+ continue;
+
+ std::string filePath = path + "/" + currentFile->d_name;
+ stat(filePath.c_str(), &fileStat);
+
+ if(S_ISDIR(fileStat.st_mode))
+ {
+ readDirToHash(filePath);
+ }
+ else if(S_ISREG(fileStat.st_mode))
+ {
+ std::ifstream file(filePath, std::ios::binary);
+
+ z_stream strm;
+ strm.zalloc = Z_NULL;
+ strm.zfree = Z_NULL;
+ strm.opaque = Z_NULL;
+ deflateInit2(&strm, Z_DEFAULT_COMPRESSION, Z_DEFLATED, 31, 8,
Z_DEFAULT_STRATEGY);
+
+ auto buf = std::unique_ptr<char[]>(new char[fileStat.st_size]);
+ std::string compressedFile = "";
+ std::string uncompressedFile = "";
+ do{
+
+ file.read(&buf[0], fileStat.st_size);
+ const long unsigned int size = file.gcount();
+ if(size == 0)
+ break;
+ long unsigned int haveComp;
+ long unsigned int compSize = compressBound(size);
+ char *cbuf;
+ cbuf = (char *)calloc(compSize, sizeof(char));
+
+ strm.next_in = (unsigned char *)&buf[0];
+ strm.avail_in = size;
+ strm.avail_out = compSize;
+ strm.next_out = (unsigned char *)&cbuf[0];
+
+ deflate(&strm, Z_FINISH);
+
+ haveComp = compSize - strm.avail_out;
+ std::string partialcompFile(cbuf, haveComp);
+ std::string partialuncompFile(buf.get(), size);
+ compressedFile = compressedFile + partialcompFile;
+ uncompressedFile = uncompressedFile + partialuncompFile;
+
+ }while(true);
+ std::pair<std::string, std::string> FilePair(uncompressedFile,
compressedFile);
+ FileHash.emplace(filePath, FilePair);
+ }
+ }
+ closedir(workingdir);
+}
+
+void FileServerRequestHandler::initializeCompression()
+{
+ std::string rootPath(get_current_dir_name());
+ static const std::vector<std::string> extensionArray
+ = {"/loleaflet/dist"};
+
+ for(const auto& extension: extensionArray)
+ {
+ std::string extensionPath;
+ extensionPath = rootPath + extension;
+ readDirToHash(extensionPath);
+ }
+}
+
+std::string FileServerRequestHandler::getCompressedFile(std::string path)
+{
+ return FileHash[path].second;
+}
+
+std::string FileServerRequestHandler::getUncompressedFile(std::string path)
+{
+ return FileHash[path].first;
+}
+
std::string FileServerRequestHandler::getRequestPathname(const HTTPRequest&
request)
{
Poco::URI requestUri(request.getURI());
@@ -256,6 +383,7 @@ std::string
FileServerRequestHandler::getRequestPathname(const HTTPRequest& requ
// Convert version back to a real file name. Remove first foreslash as the
root ends in one.
Poco::replaceInPlace(path, std::string("/loleaflet/" LOOLWSD_VERSION_HASH
"/"), std::string("loleaflet/dist/"));
+ Poco::replaceInPlace(path, std::string("/loleaflet/"),
std::string("loleaflet/"));
return path;
}
diff --git a/wsd/FileServer.hpp b/wsd/FileServer.hpp
index b27526f3..54193831 100644
--- a/wsd/FileServer.hpp
+++ b/wsd/FileServer.hpp
@@ -27,6 +27,17 @@ public:
static bool isAdminLoggedIn(const Poco::Net::HTTPRequest& request,
Poco::Net::HTTPResponse& response);
static void handleRequest(const Poco::Net::HTTPRequest& request,
Poco::MemoryInputStream& message, const std::shared_ptr<StreamSocket>& socket);
+
+ static void initializeCompression();
+
+ static void readDirToHash(std::string path);
+
+ static std::string getCompressedFile(std::string path);
+
+ static std::string getUncompressedFile(std::string path);
+
+private:
+ static std::map<std::string, std::pair<std::string, std::string>> FileHash;
};
#endif
diff --git a/wsd/LOOLWSD.cpp b/wsd/LOOLWSD.cpp
index 51a4691b..096a9ac9 100644
--- a/wsd/LOOLWSD.cpp
+++ b/wsd/LOOLWSD.cpp
@@ -794,6 +794,8 @@ void LOOLWSD::initialize(Application& self)
LOG_INF("Command trace dumping enabled to file: " << path);
}
+ FileServerRequestHandler::initializeCompression();
+
StorageBase::initialize();
ServerApplication::initialize(self);
_______________________________________________
Libreoffice-commits mailing list
libreoffice-comm...@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/libreoffice-commits
