Hello Curl hackers, I've been working with Curl library in C for some time, and I love it. Everything just works.
Recently I started using Curl to upload files from Windows machines to Amazon cloud using HTTP POST. The uploads don't perform well. They are limited by the send buffer used by Windows. On Windows 2008R2 (and probably on more recent versions as well) the system only puts on the wire the amount of bytes it has buffered. This is either what is set by SO_SNDBUF (default 8KB) or what was passed to send (CURL_MAX_WRITE_SIZE, 16KB), whichever is higher. In case of CURL uploads, the system sends 16KB, then waits for acknowledgement from the other end before asking the application for the next batch. This is readily visible in Wireshark traces. Starting with Windows 7 / 2008R2, Windows implements send buffer autotuning. This is well described here: https://msdn.microsoft.com/en-us/library/windows/desktop/bb736549(v=vs.85).aspx Theoretically on these systems the send buffer should be automatically adjusted to optimize throughput. I run a couple experiments to confirm that, and found that the send buffer is only adjusted if the socket is blocking and application buffer size is reasonably large (experiment1.c), and buffer stays at 8192 if the socket is nonblocking regardless of the application buffer size (experiment2.c). Since Curl is using nonblocking sockets, and switching to blocking would break existing functionality, an alternative approach would be to periodically run SIO_IDEAL_SEND_BACKLOG_QUERY and pass the result to SO_SNDBUF. This will work on Windows Vista SP1 / 2008 SP1 and newer, and fail silently on older Windows versions. I attached a proof-of-concept patch (autotuning.patch), which requires a reasonably recent SDK. With this patch in place I am no longer seeing upload pauses, and Windows is able to saturate the link even on high-bandwidth high-latency networks. Disclaimer: I only tested this patch on 32bit build under Visual Studio 2010 (I had to #define _WIN32_WINNT 0x0601 in order to get access to SIO_IDEAL_SEND_BACKLOG_QUERY). 64bit build currently fails on my machine (..\builds\libcurl-vc-x64-release-dll-ipv6-sspi-winssl-obj-lib/file.obj : fatal error LNK1112: module machine type 'X86' conflicts with target machine type 'x64'), I don't think this is related to my patch, but I'll keep digging. Let me know if there are chances for this patch to make its way to the next release. Thanks, Daniel
#define WIN32_LEAN_AND_MEAN #include <windows.h> #include <winsock2.h> #include <ws2tcpip.h> #include <stdlib.h> #include <stdio.h> // Need to link with Ws2_32.lib, Mswsock.lib, and Advapi32.lib #pragma comment (lib, "Ws2_32.lib") #pragma comment (lib, "Mswsock.lib") #pragma comment (lib, "AdvApi32.lib") //81920 is the default buffer size used by .NET 4.5.1 // with buffers of size 16KB and less autotuning did not work #define SEND_BUFFER_SIZE 81920 //Output with 80KB buffer: /* 8192<-current 131072<-current 262144<-current 524288<-current 1048576<-current 2097152<-current */ // code taken from https://docs.microsoft.com/en-us/windows/desktop/winsock/complete-client-code // adapted to showcase automatic buffer tuning int main(int argc, char* argv[]) { static int prev = 0; int curval=0; int curlen=sizeof(curval); char tmp[11]; WSADATA wsaData; SOCKET ConnectSocket = INVALID_SOCKET; struct addrinfo *result = NULL, *ptr = NULL, hints; char sendbuf[SEND_BUFFER_SIZE] = "PUT /research_w/en/1 HTTP/1.1\nHost: 10.51.4.147:9200\nAccept: */*\nContent-Length: 185829363\n\n"; int iResult; iResult = WSAStartup(MAKEWORD(2,2), &wsaData); if (iResult != 0) { printf("WSAStartup failed with error: %d\n", iResult); return 1; } ZeroMemory( &hints, sizeof(hints) ); hints.ai_family = AF_UNSPEC; hints.ai_socktype = SOCK_STREAM; hints.ai_protocol = IPPROTO_TCP; // Resolve the server address and port iResult = getaddrinfo("10.51.4.147", "9200", &hints, &result); if ( iResult != 0 ) { printf("getaddrinfo failed with error: %d\n", iResult); WSACleanup(); return 1; } // Attempt to connect to an address until one succeeds for(ptr=result; ptr != NULL ;ptr=ptr->ai_next) { // Create a SOCKET for connecting to server ConnectSocket = socket(ptr->ai_family, ptr->ai_socktype, ptr->ai_protocol); if (ConnectSocket == INVALID_SOCKET) { printf("socket failed with error: %ld\n", WSAGetLastError()); WSACleanup(); return 1; } // Connect to server. iResult = connect( ConnectSocket, ptr->ai_addr, (int)ptr->ai_addrlen); if (iResult == SOCKET_ERROR) { closesocket(ConnectSocket); ConnectSocket = INVALID_SOCKET; continue; } break; } freeaddrinfo(result); if (ConnectSocket == INVALID_SOCKET) { printf("Unable to connect to server!\n"); WSACleanup(); return 1; } //if you uncomment this line, Windows 2008R2 will disable send buffer autotuning //iResult = WSAIoctl(ConnectSocket,SIO_IDEAL_SEND_BACKLOG_QUERY,0,0,&curideal,sizeof(curideal),&ideallen,0,0); while(true) { iResult = send( ConnectSocket, sendbuf, (int)sizeof(sendbuf), 0 ); if (iResult == SOCKET_ERROR) { printf("send failed with error: %d\n", WSAGetLastError()); closesocket(ConnectSocket); WSACleanup(); return 1; } if(getsockopt(ConnectSocket, SOL_SOCKET, SO_SNDBUF, (char *)&curval, &curlen) == 0) { if(curval != prev) { OutputDebugStringA(itoa(curval,tmp,10)); OutputDebugStringA("<-current\n"); prev = curval; } } else OutputDebugStringA("getsockopt failed \n"); } }
#define WIN32_LEAN_AND_MEAN #include <windows.h> #include <winsock2.h> #include <ws2tcpip.h> #include <stdlib.h> #include <stdio.h> // Need to link with Ws2_32.lib, Mswsock.lib, and Advapi32.lib #pragma comment (lib, "Ws2_32.lib") #pragma comment (lib, "Mswsock.lib") #pragma comment (lib, "AdvApi32.lib") //81920 is the default buffer size used by .NET 4.5.1 // with nonblocking sockets autotuning did not work regardless of application buffer size #define SEND_BUFFER_SIZE 81920 //Output with 80KB buffer: /* 8192<-current 65536<-ideal 131072<-ideal 262144<-ideal 524288<-ideal */ // code taken from https://docs.microsoft.com/en-us/windows/desktop/winsock/complete-client-code // adapted to showcase automatic buffer tuning int main(int argc, char* argv[]) { static int prev = 0, prevideal=0; int curval=0; int curlen=sizeof(curval); DWORD ideallen; ULONG curideal=0; char tmp[11]; WSADATA wsaData; SOCKET ConnectSocket = INVALID_SOCKET; struct addrinfo *result = NULL, *ptr = NULL, hints; char sendbuf[SEND_BUFFER_SIZE] = "PUT /research_w/en/1 HTTP/1.1\nHost: 10.51.4.147:9200\nAccept: */*\nContent-Length: 185829363\n\n"; int iResult; iResult = WSAStartup(MAKEWORD(2,2), &wsaData); if (iResult != 0) { printf("WSAStartup failed with error: %d\n", iResult); return 1; } ZeroMemory( &hints, sizeof(hints) ); hints.ai_family = AF_UNSPEC; hints.ai_socktype = SOCK_STREAM; hints.ai_protocol = IPPROTO_TCP; // Resolve the server address and port iResult = getaddrinfo("10.51.4.147", "9200", &hints, &result); if ( iResult != 0 ) { printf("getaddrinfo failed with error: %d\n", iResult); WSACleanup(); return 1; } // Attempt to connect to an address until one succeeds for(ptr=result; ptr != NULL ;ptr=ptr->ai_next) { // Create a SOCKET for connecting to server ConnectSocket = socket(ptr->ai_family, ptr->ai_socktype, ptr->ai_protocol); if (ConnectSocket == INVALID_SOCKET) { printf("socket failed with error: %ld\n", WSAGetLastError()); WSACleanup(); return 1; } // Connect to server. iResult = connect( ConnectSocket, ptr->ai_addr, (int)ptr->ai_addrlen); if (iResult == SOCKET_ERROR) { closesocket(ConnectSocket); ConnectSocket = INVALID_SOCKET; continue; } break; } freeaddrinfo(result); if (ConnectSocket == INVALID_SOCKET) { printf("Unable to connect to server!\n"); WSACleanup(); return 1; } unsigned long flags = 1UL; iResult = ioctlsocket(ConnectSocket, FIONBIO, &flags); if (iResult == SOCKET_ERROR) { printf("failed to set nonblocking mode: %ld\n", WSAGetLastError()); WSACleanup(); return 1; } while(true) { iResult = send( ConnectSocket, sendbuf, (int)sizeof(sendbuf), 0 ); //ignore send failures; they don't matter for the experiment if(getsockopt(ConnectSocket, SOL_SOCKET, SO_SNDBUF, (char *)&curval, &curlen) == 0) { if(curval != prev) { OutputDebugStringA(itoa(curval,tmp,10)); OutputDebugStringA("<-current\n"); prev = curval; } } else OutputDebugStringA("getsockopt failed \n"); //with nonblocking socket it doesn't matter if we query - autotuning is disabled even if we don't iResult = WSAIoctl(ConnectSocket,SIO_IDEAL_SEND_BACKLOG_QUERY,0,0,&curideal,sizeof(curideal),&ideallen,0,0); if(iResult == 0) { if(curideal != prevideal) { OutputDebugStringA(itoa(curideal,tmp,10)); OutputDebugStringA("<-ideal\n"); prevideal = curideal; } } else { OutputDebugStringA(itoa(iResult,tmp,10)); OutputDebugStringA("WsaIOctl failed \n"); } } }
0001-Implement-send-buffer-tuning-under-Windows.patch
Description: Binary data
------------------------------------------------------------------- Unsubscribe: https://cool.haxx.se/list/listinfo/curl-library Etiquette: https://curl.haxx.se/mail/etiquette.html
