Used multiple threads to connect to the same server (httpcomponents-client
4.0.1). Each thread represents an account on the server and has different
username/password combination, so different cookies. Normally 1000 threads are
used. The problem is, even though I configured the ThreadSafeClientConnManager
to use 2000 connections, it seems that it only used 2 to 3 connections at the
same time (I used fiddler to check it). 1000 requests (one request per thread)
takes about 500 seconds to complete. Unacceptable performance. Most probable
reason is incorrect configuration. I'm new to network programming and
httpcomponents, and I can not figure out and solve the problem. Your help is
much appreciated.
The following is a class I use to communicate with the server. Each thread has
an instance of this class.
public class ServerCommunicator {
ServerCommunicator(String baseUrl) {
if (!httpClientInitialized)
initializeHttpClient();
this.baseUrl = baseUrl;
this.context = new BasicHttpContext();
this.cookieStore = new BasicCookieStore();
this.context.setAttribute(ClientContext.COOKIE_STORE,
this.cookieStore);
}
public void setBaseUrl(String baseUrl) {
this.baseUrl = baseUrl;
}
public void readCookies(String cookieFile) throws NumberFormatException,
IOException {
String name = null;
String value = null;
String comment = null;
// String commentUrl = null;
Date expiryDate = null;
// boolean isPersistent = false;
String domain = null;
String path = null;
int[] ports = null;
boolean isSecure = false;
int version = 0;
BufferedReader reader = new BufferedReader(new
FileReader(cookieFile));
String line;
while ((line = reader.readLine()) != null) {
String[] cols = line.split(",");
name = cols[0].length() == 0 ? null : cols[0];
value = cols[1].length() == 0 ? null : cols[1];
comment = cols[2].length() == 0 ? null : cols[2];
// commentUrl = cols[3].length() == 0 ? null : cols[3];
expiryDate = cols[4].length() == 0 ? null : new
Date(Long
.parseLong(cols[4]));
// isPersistent = Boolean.parseBoolean(cols[5]);
domain = cols[6].length() == 0 ? null : cols[6];
path = cols[7].length() == 0 ? null : cols[7];
ports = null;
if (cols[8].length() != 0) {
String[] strPorts = cols[8].split(" ");
ports = new int[strPorts.length];
for (int i = 0; i < strPorts.length; i++)
ports[i] =
Integer.parseInt(strPorts[i]);
}
isSecure = Boolean.parseBoolean(cols[9]);
version = Integer.parseInt(cols[10]);
BasicClientCookie cookie = new BasicClientCookie(name,
value);
cookie.setComment(comment);
cookie.setDomain(domain);
cookie.setExpiryDate(expiryDate);
cookie.setPath(path);
cookie.setSecure(isSecure);
cookie.setVersion(version);
this.cookieStore.addCookie(cookie);
}
reader.close();
}
public void writeCookies(String cookieFile) throws IOException {
int[] ports = null;
BufferedWriter writer = new BufferedWriter(new
FileWriter(cookieFile));
StringBuilder line = new StringBuilder(256);
List<Cookie> cookies = this.cookieStore.getCookies();
for (Cookie cookie : cookies) {
line.append(cookie.getName());
line.append(',');
if (cookie.getValue() != null)
line.append(cookie.getValue());
line.append(',');
if (cookie.getComment() != null)
line.append(cookie.getComment());
line.append(',');
if (cookie.getCommentURL() != null)
line.append(cookie.getCommentURL());
line.append(',');
if (cookie.getExpiryDate() != null)
line.append(cookie.getExpiryDate().getTime());
line.append(',');
line.append(cookie.isPersistent());
line.append(',');
if (cookie.getDomain() != null)
line.append(cookie.getDomain());
line.append(',');
if (cookie.getPath() != null)
line.append(cookie.getPath());
line.append(',');
ports = cookie.getPorts();
if (ports != null) {
line.append(ports[0]);
for (int i = 1; i < ports.length; i++) {
line.append(' ');
line.append(ports[i]);
}
}
line.append(',');
line.append(cookie.isSecure());
line.append(',');
line.append(cookie.getVersion());
writer.write(line.toString());
writer.newLine();
line.delete(0, line.length());
}
writer.close();
}
public synchronized byte[] getData(String path)
throws ClientProtocolException, IOException {
String url = getUrl(path);
HttpGet getReq = new HttpGet(url);
HttpResponse resp = httpClient.execute(getReq, this.context);
statusCode = resp.getStatusLine().getStatusCode();
HttpEntity entity = resp.getEntity();
if (entity != null) {
return EntityUtils.toByteArray(entity);
} else {
return null;
}
}
public synchronized String get(String path) throws
ClientProtocolException,
IOException {
String url = getUrl(path);
HttpGet getReq = new HttpGet(url);
HttpResponse resp = httpClient.execute(getReq, this.context);
statusCode = resp.getStatusLine().getStatusCode();
HttpEntity entity = resp.getEntity();
if (entity != null) {
return EntityUtils.toString(entity);
} else {
return null;
}
}
public synchronized String post(String path, String content)
throws ClientProtocolException, IOException {
String url = getUrl(path);
HttpPost postReq = new HttpPost(url);
StringEntity postEnt = new StringEntity(content);
postEnt.setContentType("application/x-www-form-urlencoded");
postReq.setEntity(postEnt);
HttpResponse resp = httpClient.execute(postReq, this.context);
statusCode = resp.getStatusLine().getStatusCode();
HttpEntity entity = resp.getEntity();
if (entity != null) {
return EntityUtils.toString(entity);
} else {
return null;
}
}
public synchronized String post(String path, List<NameValuePair> params)
throws ClientProtocolException, IOException {
String url = getUrl(path);
HttpPost postReq = new HttpPost(url);
UrlEncodedFormEntity postEnt = new UrlEncodedFormEntity(params,
HTTP.UTF_8);
postEnt.setContentType("application/x-www-form-urlencoded");
postReq.setEntity(postEnt);
HttpResponse resp = httpClient.execute(postReq, this.context);
statusCode = resp.getStatusLine().getStatusCode();
HttpEntity entity = resp.getEntity();
if (entity != null) {
return EntityUtils.toString(entity);
} else {
return null;
}
}
public static void main(String[] args) {
}
private String getUrl(String path) {
return path.startsWith("http://";) ? path : baseUrl + path;
}
private int statusCode = 0;
public int getStatusCode() {
return statusCode;
}
private String baseUrl = null;
private HttpContext context = null;
private CookieStore cookieStore = null;
private static HttpClient httpClient = null;
private static String proxyHost = null;
private static int proxyPort = 80;
private static boolean httpClientInitialized = false;
public static void setProxy(String host, int port) {
proxyHost = host;
proxyPort = port;
}
private static void initializeHttpClient() {
if (!httpClientInitialized) {
// Create and initialize HTTP parameters
HttpParams params = new BasicHttpParams();
ConnManagerParams.setMaxTotalConnections(params, 2000);
// ConnManagerParams.setTimeout(params, 10000); // in
millisecs
HttpProtocolParams.setVersion(params,
HttpVersion.HTTP_1_1);
// Create and initialize scheme registry
SchemeRegistry schemeRegistry = new SchemeRegistry();
schemeRegistry.register(new Scheme("http",
PlainSocketFactory
.getSocketFactory(), 80));
ClientConnectionManager cm = new
ThreadSafeClientConnManager(
params, schemeRegistry);
httpClient = new DefaultHttpClient(cm, params);
if (proxyHost != null) {
httpClient.getParams().setParameter(
ConnRoutePNames.DEFAULT_PROXY,
new HttpHost(proxyHost,
proxyPort));
}
httpClient.getParams().setParameter("http.useragent",
"Mozilla/4.0 (compatible; MSIE 6.1;
Windows XP)");
httpClient.getParams().setParameter("http.protocol.allow-circular-
redirects", true);
httpClientInitialized = true;
}
}
public static void shutdown() {
httpClient.getConnectionManager().shutdown();
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
