Author: suokko
Date: Wed May 28 22:10:57 2008
New Revision: 26902
URL: http://svn.gna.org/viewcvs/wesnoth?rev=26902&view=rev
Log:
* Implemented send_file to reduce memory usage when sending files
Modified:
branches/1.4/changelog
branches/1.4/src/campaign_server/campaign_server.cpp
branches/1.4/src/network.cpp
branches/1.4/src/network.hpp
branches/1.4/src/network_worker.cpp
branches/1.4/src/network_worker.hpp
Modified: branches/1.4/changelog
URL:
http://svn.gna.org/viewcvs/wesnoth/branches/1.4/changelog?rev=26902&r1=26901&r2=26902&view=diff
==============================================================================
--- branches/1.4/changelog (original)
+++ branches/1.4/changelog Wed May 28 22:10:57 2008
@@ -21,6 +21,7 @@
(bug #11386)
* Backported gracefull restart feature to server
* Improved connection stability with slow connections.
+ * Implemented send_file to reduce memory usage when sending files
* Fixed a mememory leak in networking code
* fixed an alignement issue which caused a SIGBUS on a Sparc
(debian bug #426318)
Modified: branches/1.4/src/campaign_server/campaign_server.cpp
URL:
http://svn.gna.org/viewcvs/wesnoth/branches/1.4/src/campaign_server/campaign_server.cpp?rev=26902&r1=26901&r2=26902&view=diff
==============================================================================
--- branches/1.4/src/campaign_server/campaign_server.cpp (original)
+++ branches/1.4/src/campaign_server/campaign_server.cpp Wed May 28 22:10:57
2008
@@ -381,15 +381,17 @@
network::send_data(construct_error("Add-on '" + (*req)["name"] + "'not
found."), sock, gzipped);
} else {
- size_t size =
file_size((*campaign)["filename"]);
- scoped_istream stream =
istream_file((*campaign)["filename"]);
if (gzipped)
{
+
network::send_file((*campaign)["filename"], sock);
+#if 0
util::scoped_resource<char*,util::delete_array> buf(new char[size+1]);
stream->read(buf,size);
network::send_raw_data(buf, size, sock);
+#endif
} else {
+ scoped_istream
stream = istream_file((*campaign)["filename"]);
config cfg;
read_gz(cfg,
*stream);
network::send_data(cfg, sock, false);
@@ -510,8 +512,8 @@
add_license(*data);
- scoped_ostream
campaign_file = ostream_file(filename);
{
+ scoped_ostream
campaign_file = ostream_file(filename);
config_writer
writer(*campaign_file, true, "",compress_level_);
writer.write(*data);
}
@@ -598,8 +600,8 @@
std::string scripts =
validate_all_python_scripts(campaign_file);
if (!scripts.empty()) {
// Write the
campaign with changed filenames back to disk
- scoped_ostream
ostream = ostream_file((*campaign)["filename"]);
{
+
scoped_ostream ostream = ostream_file((*campaign)["filename"]);
config_writer writer(*ostream, true, "",compress_level_);
writer.write(campaign_file);
}
Modified: branches/1.4/src/network.cpp
URL:
http://svn.gna.org/viewcvs/wesnoth/branches/1.4/src/network.cpp?rev=26902&r1=26901&r2=26902&view=diff
==============================================================================
--- branches/1.4/src/network.cpp (original)
+++ branches/1.4/src/network.cpp Wed May 28 22:10:57 2008
@@ -819,6 +819,24 @@
return result;
}
+void send_file(const std::string& filename, connection connection_num)
+{
+ assert(connection_num > 0);
+ if(bad_sockets.count(connection_num) || bad_sockets.count(0)) {
+ return;
+ }
+
+ const connection_map::iterator info = connections.find(connection_num);
+ if (info == connections.end()) {
+ ERR_NW << "Error: socket: " << connection_num
+ << "\tnot found in connection_map. Not sending...\n";
+ return;
+ }
+
+ network_worker_pool::queue_file(info->second.sock, filename);
+
+}
+
//! @todo Note the gzipped parameter should be removed later, we want to send
//! all data gzipped. This can be done once the campaign server is also updated
//! to work with gzipped data.
Modified: branches/1.4/src/network.hpp
URL:
http://svn.gna.org/viewcvs/wesnoth/branches/1.4/src/network.hpp?rev=26902&r1=26901&r2=26902&view=diff
==============================================================================
--- branches/1.4/src/network.hpp (original)
+++ branches/1.4/src/network.hpp Wed May 28 22:10:57 2008
@@ -133,6 +133,8 @@
connection receive_data(config& cfg, connection connection_num, unsigned int
timeout);
connection receive_data(std::vector<char>& buf);
+void send_file(const std::string&, connection);
+
//! Function to send data down a given connection,
//! or broadcast to all peers if connection_num is 0.
//! Throws error.
Modified: branches/1.4/src/network_worker.cpp
URL:
http://svn.gna.org/viewcvs/wesnoth/branches/1.4/src/network_worker.cpp?rev=26902&r1=26901&r2=26902&view=diff
==============================================================================
--- branches/1.4/src/network_worker.cpp (original)
+++ branches/1.4/src/network_worker.cpp Wed May 28 22:10:57 2008
@@ -25,6 +25,7 @@
#include "network_worker.hpp"
#include "network.hpp"
#include "thread.hpp"
+#include "filesystem.hpp"
//#include "wesconfig.h"
#include "serialization/binary_or_text.hpp"
#include "serialization/binary_wml.hpp"
@@ -38,6 +39,10 @@
#include <iostream>
#include <map>
#include <vector>
+
+#ifdef USE_SENDFILE
+#include <sys/sendfile.h>
+#endif
#include <boost/iostreams/filter/gzip.hpp>
@@ -277,13 +282,21 @@
buf.back() = 0;
}
-static SOCKET_STATE send_buffer(TCPsocket sock, std::vector<char>& buf)
+
+
+static SOCKET_STATE send_buffer(TCPsocket sock, std::vector<char>& buf, int
in_size = -1)
{
#ifdef __BEOS__
int timeout = 15000;
#endif
size_t upto = 0;
+
size_t size = buf.size();
+ if (in_size != -1)
+ {
+ size = in_size;
+ }
+
if (!raw_data_only)
{
@@ -365,6 +378,43 @@
return SOCKET_ERRORED;
}
+}
+
+static SOCKET_STATE send_file(buffer* buf)
+{
+ size_t upto = 0;
+ int send_size = 0;
+ size_t filesize = file_size(buf->config_error);
+
+#ifdef USE_SENDFILE
+ std::vector<char>& buffer;
+ buffer.reserve(4);
+ SDLNet_Write32(filesize,&buffer[0]);
+#else
+ SDLNet_Write32(filesize,&buf->raw_buffer[0]);
+ scoped_istream file_stream = istream_file(buf->config_error);
+ send_buffer(buf->sock, buf->raw_buffer, 4);
+#endif
+ while (true)
+ {
+#ifdef USE_SENDFILE
+#else
+ // read data
+ file_stream->read(&buf->raw_buffer[0], buf->raw_buffer.size());
+ send_size = file_stream->gcount();
+ upto += send_size;
+ // send data to socket
+ send_buffer(buf->sock, buf->raw_buffer, send_size);
+ if (upto == filesize)
+ {
+ buf->raw_buffer[0] = 0;
+ send_buffer(buf->sock, buf->raw_buffer, 1);
+ break;
+ }
+
+#endif
+ }
+ return SOCKET_READY;
}
static SOCKET_STATE receive_buf(TCPsocket sock, std::vector<char>& buf)
@@ -509,13 +559,19 @@
std::vector<char> buf;
if(sent_buf) {
- if(sent_buf->raw_buffer.empty()) {
- output_to_buffer(sent_buf->sock,
sent_buf->config_buf, sent_buf->stream, sent_buf->gzipped);
- const std::string &value =
sent_buf->stream.str();
- make_network_buffer(value.c_str(),
value.size(), sent_buf->raw_buffer);
- }
-
- result = send_buffer(sent_buf->sock,
sent_buf->raw_buffer);
+ if(!sent_buf->config_error.empty())
+ {
+ // We have file to send over net
+ send_file(sent_buf);
+ } else {
+ if(sent_buf->raw_buffer.empty()) {
+ output_to_buffer(sent_buf->sock,
sent_buf->config_buf, sent_buf->stream, sent_buf->gzipped);
+ const std::string &value =
sent_buf->stream.str();
+ make_network_buffer(value.c_str(),
value.size(), sent_buf->raw_buffer);
+ }
+
+ result = send_buffer(sent_buf->sock,
sent_buf->raw_buffer);
+ }
delete sent_buf;
} else {
result = receive_buf(sock,buf);
@@ -726,11 +782,8 @@
return res;
}
-void queue_raw_data(TCPsocket sock, const char* buf, int len)
-{
- buffer* queued_buf = new buffer(sock);
- assert(*buf == 31);
- make_network_buffer(buf, len, queued_buf->raw_buffer);
+static void queue_buffer(TCPsocket sock, buffer* queued_buf)
+{
const size_t shard = get_shard(sock);
const threading::lock lock(*shard_mutexes[shard]);
outgoing_bufs[shard].push_back(queued_buf);
@@ -738,7 +791,27 @@
if(i->second == SOCKET_READY || i->second == SOCKET_ERRORED) {
cond[shard]->notify_one();
}
-
+}
+
+void queue_file(TCPsocket sock, const std::string& filename)
+{
+ buffer* queued_buf = new buffer(sock);
+ queued_buf->config_error = filename;
+#ifndef USE_SENDFILE
+ // We reserve buffer in main thread
+ // this helps in memory problems with threads
+ // We use 8KB buffer
+ queued_buf->raw_buffer.resize(1024*8);
+#endif
+ queue_buffer(sock, queued_buf);
+}
+
+void queue_raw_data(TCPsocket sock, const char* buf, int len)
+{
+ buffer* queued_buf = new buffer(sock);
+ assert(*buf == 31);
+ make_network_buffer(buf, len, queued_buf->raw_buffer);
+ queue_buffer(sock, queued_buf);
}
void queue_data(TCPsocket sock,const config& buf, const bool gzipped)
@@ -749,18 +822,7 @@
queued_buf->config_buf = buf;
// output_to_buffer(sock, buf, queued_buf->stream, gzipped);
queued_buf->gzipped = gzipped;
- {
- const size_t shard = get_shard(sock);
- const threading::lock lock(*shard_mutexes[shard]);
-
- outgoing_bufs[shard].push_back(queued_buf);
-
- socket_state_map::const_iterator i =
sockets_locked[shard].insert(std::pair<TCPsocket,SOCKET_STATE>(sock,SOCKET_READY)).first;
- if(i->second == SOCKET_READY || i->second == SOCKET_ERRORED) {
- cond[shard]->notify_one();
- }
- }
-
+ queue_buffer(sock, queued_buf);
}
namespace
Modified: branches/1.4/src/network_worker.hpp
URL:
http://svn.gna.org/viewcvs/wesnoth/branches/1.4/src/network_worker.hpp?rev=26902&r1=26901&r2=26902&view=diff
==============================================================================
--- branches/1.4/src/network_worker.hpp (original)
+++ branches/1.4/src/network_worker.hpp Wed May 28 22:10:57 2008
@@ -46,6 +46,8 @@
TCPsocket get_received_data(TCPsocket sock, config& cfg, bool* gzipped = 0);
TCPsocket get_received_data(std::vector<char>& buf);
+void queue_file(TCPsocket sock, const std::string&);
+
void queue_raw_data(TCPsocket sock, const char* buf, int len);
void queue_data(TCPsocket sock, const config& buf, const bool gzipped);
bool is_locked(const TCPsocket sock);
_______________________________________________
Wesnoth-commits mailing list
[email protected]
https://mail.gna.org/listinfo/wesnoth-commits
