liupc commented on issue #23602: [SPARK-26674][CORE]Consolidate CompositeByteBuf when reading large frame URL: https://github.com/apache/spark/pull/23602#issuecomment-459666951 The benchmark codes is as below: ``` private CompositeByteBuf createCompositeBuf(ByteBufAllocator alloc, int numComponents, int size, int writtenBytes) { CompositeByteBuf compositeByteBuf = alloc.compositeBuffer(Integer.MAX_VALUE); for (int i =0; i< numComponents; i++) { ByteBuf buf = alloc.ioBuffer(size); buf.writerIndex(writtenBytes); compositeByteBuf.addComponent(buf).writerIndex(compositeByteBuf.writerIndex() + buf.readableBytes()); } return compositeByteBuf; } private void testConsolidateWithLoop(String testName, ByteBufAllocator alloc, int numComponents, int size, int util, int loopCount) { long totalTime = 0L; int writtenBytes = (int)((double)size * util / 100); for (int i = 0; i < loopCount; i++) { CompositeByteBuf buf = createCompositeBuf(alloc, numComponents, size, writtenBytes); long start = System.currentTimeMillis(); buf.consolidate(); long cost = System.currentTimeMillis() - start; totalTime += cost; buf.release(); } System.out.println("[" + testName + "]"); System.out.println("Allocating " + writtenBytes + " bytes"); System.out.println("Time cost with " + loopCount + " loop for consolidating: " + totalTime + " millis"); System.out.println(); } @Test public void benchmarkForConsolidation() throws Exception { PooledByteBufAllocator alloc = new PooledByteBufAllocator(true); testConsolidateWithLoop("test consolidate 100 buffers each with 10m, 50% used for 1 loop", alloc, 100, 1024 * 1024 * 10, 50, 1); testConsolidateWithLoop("test consolidate 100 buffers each with 10m, 100% used for 1 loop", alloc, 100, 1024 * 1024 * 10, 100, 1); testConsolidateWithLoop("test consolidate 100 buffers each with 10m, 50% used for 10 loop", alloc, 100, 1024 * 1024 * 10, 50, 10); testConsolidateWithLoop("test consolidate 100 buffers each with 10m, 100% used for 10 loop", alloc, 100, 1024 * 1024 * 10, 100, 10); testConsolidateWithLoop("test consolidate 100 buffers each with 10m, 50% used for 50 loop", alloc, 100, 1024 * 1024 * 10, 50, 50); testConsolidateWithLoop("test consolidate 100 buffers each with 10m, 100% used for 50 loop", alloc, 100, 1024 * 1024 * 10, 100, 50); ///////// testConsolidateWithLoop("test consolidate 20 buffers each with 50m, 50% used for 1 loop", alloc, 20, 1024 * 1024 * 50, 50, 1); testConsolidateWithLoop("test consolidate 20 buffers each with 50m, 100% used for 1 loop", alloc, 20, 1024 * 1024 * 50, 100, 1); testConsolidateWithLoop("test consolidate 20 buffers each with 50m, 50% used for 10 loop", alloc, 20, 1024 * 1024 * 50, 50, 10); testConsolidateWithLoop("test consolidate 20 buffers each with 50m, 100% used for 10 loop", alloc, 20, 1024 * 1024 * 50, 100, 10); testConsolidateWithLoop("test consolidate 20 buffers each with 50m, 50% used for 50 loop", alloc, 20, 1024 * 1024 * 50, 50, 50); testConsolidateWithLoop("test consolidate 20 buffers each with 50m, 100% used for 50 loop", alloc, 20, 1024 * 1024 * 50, 100, 50); ////// testConsolidateWithLoop("test consolidate 10 buffers each with 100m, 50% used for 1 loop", alloc, 10, 1024 * 1024 * 100, 50, 1); testConsolidateWithLoop("test consolidate 10 buffers each with 100m, 100% used for 1 loop", alloc, 10, 1024 * 1024 * 100, 100, 1); testConsolidateWithLoop("test consolidate 10 buffers each with 100m, 50% used for 10 loop", alloc, 10, 1024 * 1024 * 100, 50, 10); testConsolidateWithLoop("test consolidate 10 buffers each with 100m, 100% used for 10 loop", alloc, 10, 1024 * 1024 * 100, 100, 10); testConsolidateWithLoop("test consolidate 10 buffers each with 100m, 50% used for 50 loop", alloc, 10, 1024 * 1024 * 100, 50, 50); testConsolidateWithLoop("test consolidate 10 buffers each with 100m, 100% used for 50 loop", alloc, 10, 1024 * 1024 * 100, 100, 50); } ```
---------------------------------------------------------------- This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: [email protected] With regards, Apache Git Services --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
