[?], thanks. I did more experiments. It looks it's not http client lib but more like network IO task scheduling on OS although we didn't do anything special. Thanks anyway.
On Tue, Oct 15, 2013 at 1:58 PM, Oleg Kalnichevski <[email protected]> wrote: > On Tue, 2013-10-15 at 11:17 +0100, Ke Ren wrote: > > yes, that was with http client 4.2.5. I tried to see if there is any > > different with 4.3. The following is the code with 4.2.5 > > > > import concurrent.{ExecutionContext, Await, Future, Promise} > > import scala.concurrent.duration._ > > import java.util.concurrent.Executors > > import org.apache.http.impl.client.{DefaultHttpClient, HttpClients} > > import org.apache.http.util.EntityUtils > > import org.apache.http.conn.scheme.{PlainSocketFactory, Scheme, > > SchemeRegistry} > > import org.apache.http.conn.ssl.SSLSocketFactory > > import org.apache.http.impl.conn.PoolingClientConnectionManager > > import org.apache.http.params.SyncBasicHttpParams > > import org.apache.http.client.methods.HttpGet > > > > object MainTest { > > def main(args: Array[String]) { > > var num = 100 > > var jobs = 1000 > > if (args.size == 2) { > > num = args(0).toInt > > jobs = args(1).toInt > > } > > val executorService = Executors.newCachedThreadPool() > > implicit val ec = > ExecutionContext.fromExecutorService(executorService) > > val schemeRegistry = new SchemeRegistry(); > > schemeRegistry.register(new Scheme("http", 80, > > PlainSocketFactory.getSocketFactory())); > > schemeRegistry.register(new Scheme("https", 443, > > SSLSocketFactory.getSocketFactory())); > > val mgr = new PoolingClientConnectionManager(schemeRegistry); > > val params = new SyncBasicHttpParams(); > > val httpclient = new DefaultHttpClient(mgr, params); > > > > var workers = List.empty[Worker] > > var futures = List.empty[Future[Boolean]] > > for (i <- 0 to num) { > > val p = Promise[Boolean] > > workers = workers.::(new Worker(httpclient, p, jobs)) > > futures = futures.::(p.future) > > } > > > > val allF = futures.tail.foldLeft(futures.head) { > > case (future, rowFuture) => future.flatMap { > > status => > > rowFuture.map(finished => status && finished) > > } > > } > > > > workers foreach { > > worker => > > worker.start() > > } > > > > Await.result(allF, 1 hour) > > println("finished") > > } > > } > > > > class Worker(httpclient: DefaultHttpClient, p: Promise[Boolean], jobs: > Int) > > extends Thread { > > override def run { > > for (i <- 0 to jobs) { > > try { > > var httpget = new HttpGet("xxxx") > > val response = httpclient.execute(httpget); > > try { > > val entity = response.getEntity(); > > if (entity != null) { > > EntityUtils.toByteArray(entity) > > } > > } finally { > > response.close(); > > httpclient.close() > > } > > } catch { > > case e: Throwable => > > e.printStackTrace() > > } > > } > > p.success(true) > > } > > } > > I see nothing wrong with your code. I am not familiar with Scala though. > HttpClient related aspects look sane to me. > > Oleg > > > > --------------------------------------------------------------------- > To unsubscribe, e-mail: [email protected] > For additional commands, e-mail: [email protected] > >
