Memory leak - java heap space exception
---------------------------------------
Key: HTTPCLIENT-949
URL: https://issues.apache.org/jira/browse/HTTPCLIENT-949
Project: HttpComponents HttpClient
Issue Type: Bug
Components: HttpClient
Affects Versions: 4.0.1
Environment: Xp Sp3 on Intel Duo
Reporter: Andrea Braia
Priority: Critical
I use HttpComponents 4.0.1 and i have an application that has to do about 20
milions of requests, normal get and post request.
I suddenly engaged a memory leak problem... i found several post where many
users has the same problem, and they solve it in several ways... but no one
seem to be the right one.
The problem itself seems to be with Java Strings and the size of big String
handling.
For testing memory leak i used for retrive pages google.it, as you can see
below
*****************
//test class
public void test_HttpRequest(){
MyUtil.MyApp.AppHandler app_handler = new MyUtil.MyApp.AppHandler();
for(int i=0; i<10000; i++){
HttpGet httpget = new HttpGet("http://www.google.it/";);
// HttpGet httpget = new HttpGet("http://www.comuni-italiani.it/";);
httpget.setHeader("User-Agent", "Mozilla/5.0 (Windows; U; Windows NT 5.1;
en-US; rv:1.8.1) Gecko/20061010 Firefox/2.0 (.NET CLR 3.5.30729)");
System.out.println("processing request num: "+(i+1));
new C_HttpRequestManager("google.it",
80,
httpget,
true,
5000,
false,//no proxy
null,
0,
app_handler).getResponse();
try {
System.out.println("waiting 2 s...");
Thread.sleep(2000);
} catch (InterruptedException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
System.out.println("requests completed, insert value...");
MyUtil.MyIO.Util.getLineFromStdInput();
}
*****************
now i show you the code about the http class request...
*****************
//C_HttpRequestManager
package MyIO;
import java.io.IOException;
import java.io.InputStreamReader;
import org.apache.http.HttpEntity;
import org.apache.http.HttpHost;
import org.apache.http.HttpVersion;
import org.apache.http.ParseException;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.methods.HttpRequestBase;
import org.apache.http.conn.params.ConnRoutePNames;
import org.apache.http.conn.scheme.PlainSocketFactory;
import org.apache.http.conn.scheme.Scheme;
import org.apache.http.conn.scheme.SchemeRegistry;
import org.apache.http.conn.ssl.SSLSocketFactory;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.impl.conn.SingleClientConnManager;
import org.apache.http.params.BasicHttpParams;
import org.apache.http.params.CoreConnectionPNames;
import org.apache.http.params.HttpParams;
import org.apache.http.params.HttpProtocolParams;
import org.apache.http.util.EntityUtils;
import BaseExcp.BaseExcp;
public class C_HttpRequestManager extends A_HttpRequestManager{
public final String class_name = "C_HttpRequestManager";
protected boolean keep_try;
protected int time_out;//ms, if zero use default
public C_HttpRequestManager(HttpParams _params, SchemeRegistry
_supportedSchemes, HttpHost _target_host, HttpRequestBase _request, boolean
_keep_try, int _time_out, MyUtil.MyApp.AppHandler _h_app){
this.params = _params;
this.supportedSchemes = _supportedSchemes;
this.target_host = _target_host;
this.request = _request;
this.keep_try = _keep_try;
this.time_out = _time_out;
if(time_out != 0){
this.params.setParameter(CoreConnectionPNames.SO_TIMEOUT ,
this.time_out);
}else{//use default timeout
this.params.setParameter(CoreConnectionPNames.SO_TIMEOUT , this.TIMEOUT);
}
this.app_handler = _h_app;
this.app_handler.addAppHandled(this);
}
public C_HttpRequestManager(String _target_host_name, int
_target_host_port, HttpRequestBase _request, boolean _keep_try, int
_time_out, boolean use_proxy, String proxy_host_name, int proxy_host_port,
MyUtil.MyApp.AppHandler _h_app){
this.params = new BasicHttpParams();
HttpProtocolParams.setVersion(this.params, HttpVersion.HTTP_1_1);
HttpProtocolParams.setContentCharset(this.params, "UTF-8");
HttpProtocolParams.setUseExpectContinue(this.params, true);
this.supportedSchemes = new SchemeRegistry();
// Register the "http" and "https" protocol schemes, they are
// required by the default operator to look up socket factories.
this.supportedSchemes.register(new Scheme("http",
PlainSocketFactory.getSocketFactory(), 80));
this.supportedSchemes.register(new Scheme("https",
SSLSocketFactory.getSocketFactory(), 443));
this.target_host = new HttpHost(_target_host_name, _target_host_port);
this.request = _request;
this.keep_try = _keep_try;
this.time_out = _time_out;
if(time_out != 0){
this.params.setParameter(CoreConnectionPNames.SO_TIMEOUT ,
this.time_out);
}else{//use default timeout
this.params.setParameter(CoreConnectionPNames.SO_TIMEOUT , this.TIMEOUT);
}
//proxy config
if(use_proxy)
this.params.setParameter(ConnRoutePNames.DEFAULT_PROXY, new
HttpHost(proxy_host_name, proxy_host_port));
this.app_handler = _h_app;
this.app_handler.addAppHandled(this);
}
@Override
public void exitAppCarefully() {
if(ccm != null)
this.ccm.shutdown();
}
@Override
public C_HttpResponseStatusEntity getResponse() {
final String function_name = "getResponse";
final String function_name_formatted =
this.class_name+"."+function_name+": ";
C_HttpResponseStatusEntity response = null;
do{
ccm = new SingleClientConnManager(params, supportedSchemes);
httpclient = new DefaultHttpClient(ccm, params);
try {
h_response = httpclient.execute(target_host, request);
} catch (ClientProtocolException e) {
try {
if(this.app_handler.hasLog())
this.app_handler.log.writeErr(e.toString());
} catch (BaseExcp e1) {
// TODO Auto-generated catch block
e1.printStackTrace();
}
this.h_status = e.toString();
this.h_entity = e.toString();
} catch (IOException e) {
response = this.pharse_IOException(e);
if(response.getStatus().equals(this.SocketTimeoutException)){
if(this.app_handler.hasLog()){
try {
this.app_handler.log.writeInfo(function_name_formatted+SocketTimeoutException);
} catch (BaseExcp e1) {
// TODO Auto-generated catch block
e1.printStackTrace();
this.app_handler.exitAppCarefully();
}
}
continue;
}else{
return response;
}
}
this.h_status = h_response.getStatusLine().toString(); //Oleg is good to
use String? ...Should I?
//Oleg, here starts the
unsafe code, as learnd by posts found on internet... so below this commented
code starts the new code modified using StringBuffer and Streams....
//
// try {
// this.h_entity = EntityUtils.toString(this.h_response.getEntity());
//
// } catch (ParseException e) {
//
// try {
// if(this.app_handler.hasLog())
// this.app_handler.log.writeErr(e.toString());
// } catch (BaseExcp e1) {
// // TODO Auto-generated catch block
// e1.printStackTrace();
// }
//
// this.h_status = e.toString();
// this.h_entity = e.toString();
//
// } catch (IOException e) {
//
// response = this.pharse_IOException(e);
//
// if(response.getStatus().equals(this.SocketTimeoutException)){
// if(this.app_handler.hasLog()){
// try {
//this.app_handler.log.writeInfo(function_name_formatted+SocketTimeoutException);
// } catch (BaseExcp e1) {
// // TODO Auto-generated catch block
// e1.printStackTrace();
// this.app_handler.exitAppCarefully();
// }
// }
// continue;
// }else{
//
// return response;
// }
// }
// Get hold of the response entity
HttpEntity entity = this.h_response.getEntity();
// If the response does not enclose an entity, there is no need
// to bother about connection release
if (entity != null) {
InputStreamReader ir = null;
try {
ir = new InputStreamReader(entity.getContent());
} catch (IllegalStateException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
char[] cbuf = new char[1024];
StringBuffer sbuf = new StringBuffer();
int len = 0;
int tot_len = 0;
try {
while((len = ir.read(cbuf)) > 0) {
sbuf.append(cbuf, 0, len);
tot_len = tot_len + len;
}
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
System.out.println("tot_len: "+tot_len);
System.out.println("entity.getContentLength():
"+entity.getContentLength());
this.h_entity = sbuf.toString();
// Closing the input stream will trigger connection release
try {
ir.close();
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
this.request.abort();
}
//System.out.println(this.h_status);
//System.out.println(this.h_entity);
this.keep_try = false;
}while(this.keep_try);
response = new C_HttpResponseStatusEntity(this.h_status, this.h_entity,
MyIO.C_HttpResponseStatusEntity.createHeaders(h_response.getAllHeaders()));
this.ccm.shutdown(); this.ccm = null;
return response;
}
}
*****************
well, i hope to hear from you soon...
Andrea
--
This message is automatically generated by JIRA.
-
You can reply to this email to add a comment to the issue online.
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
