LuciferYang commented on PR #37999: URL: https://github.com/apache/spark/pull/37999#issuecomment-1259014121
I write a multi thread test as follows: https://github.com/LuciferYang/spark/blob/fe26455668a904400e4b81ec696b7cd3abb3c923/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/MJacksonBenchmark.scala ```scala def testWriteMapToJson(valuesPerIteration: Int, threads: Int): Unit = { val map = Map("intValue" -> 1, "longValue" -> 2L, "doubleValue" -> 3.0D, "stringValue" -> "4", "floatValue" -> 5.0F, "booleanValue" -> true) val benchmark = new Benchmark(s"Test $threads threads write map to json", valuesPerIteration, output = output) val multi = Array.fill(threads)({ val ret = new ObjectMapper() ret.registerModule(DefaultScalaModule) ret }) benchmark.addCase("Test use multi mapper") { _: Int => val latch = new CountDownLatch(valuesPerIteration) val executor = ThreadUtils.newDaemonFixedThreadPool(threads, "multi") for (i <- 0 until valuesPerIteration) { executor.submit(new Runnable { override def run(): Unit = { val idx = i % threads multi(idx).writeValueAsString(map) latch.countDown() } }) } latch.await() executor.shutdown() } val mapper = new ObjectMapper() mapper.registerModule(DefaultScalaModule) val singleton = Array.fill(threads)(mapper) benchmark.addCase("Test use singleton mapper") { _: Int => val latch = new CountDownLatch(valuesPerIteration) val executor = ThreadUtils.newDaemonFixedThreadPool(threads, "singleton") for (i <- 0 until valuesPerIteration) { executor.submit(new Runnable { override def run(): Unit = { val idx = i % threads singleton(idx).writeValueAsString(map) latch.countDown() } }) } latch.await() executor.shutdown() } benchmark.run() } def testReadJsonToMap(valuesPerIteration: Int, threads: Int): Unit = { val input = { val map = Map("intValue" -> 1, "longValue" -> 2L, "doubleValue" -> 3.0D, "stringValue" -> "4", "floatValue" -> 5.0F, "booleanValue" -> true) val mapper = new ObjectMapper() mapper.registerModule(DefaultScalaModule) mapper.writeValueAsString(map) } val benchmark = new Benchmark(s"Test $threads threads read json to map", valuesPerIteration, output = output) val multi = Array.fill(threads)({ val ret = new ObjectMapper() ret.registerModule(DefaultScalaModule) ret }) benchmark.addCase("Test use multi mapper") { _: Int => val latch = new CountDownLatch(valuesPerIteration) val executor = ThreadUtils.newDaemonFixedThreadPool(threads, "multi") for (i <- 0 until valuesPerIteration) { executor.submit(new Runnable { override def run(): Unit = { val idx = i % threads multi(idx).readValue(input, classOf[mutable.HashMap[String, String]]) latch.countDown() } }) } latch.await() executor.shutdown() } val mapper = new ObjectMapper() mapper.registerModule(DefaultScalaModule) val singleton = Array.fill(threads)(mapper) benchmark.addCase("Test use singleton mapper") { _: Int => val latch = new CountDownLatch(valuesPerIteration) val executor = ThreadUtils.newDaemonFixedThreadPool(threads, "singleton") for (i <- 0 until valuesPerIteration) { executor.submit(new Runnable { override def run(): Unit = { val idx = i % threads singleton(idx).readValue(input, classOf[mutable.HashMap[String, String]]) latch.countDown() } }) } latch.await() executor.shutdown() } benchmark.run() } ``` The result from GA as follows: ``` OpenJDK 64-Bit Server VM 1.8.0_345-b01 on Linux 5.15.0-1020-azure Intel(R) Xeon(R) Platinum 8272CL CPU @ 2.60GHz Test 5 threads read json to map: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------ Test use multi mapper 1927 1959 46 0.5 1927.2 1.0X Test use singleton mapper 1727 1884 222 0.6 1727.0 1.1X OpenJDK 64-Bit Server VM 1.8.0_345-b01 on Linux 5.15.0-1020-azure Intel(R) Xeon(R) Platinum 8272CL CPU @ 2.60GHz Test 10 threads read json to map: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------ Test use multi mapper 1411 1465 76 0.7 1411.2 1.0X Test use singleton mapper 1297 1322 36 0.8 1296.7 1.1X OpenJDK 64-Bit Server VM 1.8.0_345-b01 on Linux 5.15.0-1020-azure Intel(R) Xeon(R) Platinum 8272CL CPU @ 2.60GHz Test 20 threads read json to map: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------ Test use multi mapper 1570 1598 41 0.6 1569.7 1.0X Test use singleton mapper 1433 1455 31 0.7 1432.7 1.1X OpenJDK 64-Bit Server VM 1.8.0_345-b01 on Linux 5.15.0-1020-azure Intel(R) Xeon(R) Platinum 8272CL CPU @ 2.60GHz Test 5 threads write map to json: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------ Test use multi mapper 804 948 183 1.2 803.7 1.0X Test use singleton mapper 778 954 167 1.3 777.7 1.0X OpenJDK 64-Bit Server VM 1.8.0_345-b01 on Linux 5.15.0-1020-azure Intel(R) Xeon(R) Platinum 8272CL CPU @ 2.60GHz Test 10 threads write map to json: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------ Test use multi mapper 784 865 75 1.3 784.0 1.0X Test use singleton mapper 777 954 179 1.3 776.6 1.0X OpenJDK 64-Bit Server VM 1.8.0_345-b01 on Linux 5.15.0-1020-azure Intel(R) Xeon(R) Platinum 8272CL CPU @ 2.60GHz Test 20 threads write map to json: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------ Test use multi mapper 777 841 96 1.3 777.4 1.0X Test use singleton mapper 782 932 134 1.3 782.1 1.0X ``` and I use a bare metal server to test more threads, the test result as follows: ``` OpenJDK 64-Bit Server VM 1.8.0_312-b07 on Linux 4.14.0_1-0-0-45 Intel(R) Xeon(R) Gold 6271C CPU @ 2.60GHz Test 5 threads read json to map: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------ Test use multi mapper 1197 1338 200 0.8 1197.1 1.0X Test use singleton mapper 1023 1277 360 1.0 1022.8 1.2X OpenJDK 64-Bit Server VM 1.8.0_312-b07 on Linux 4.14.0_1-0-0-45 Intel(R) Xeon(R) Gold 6271C CPU @ 2.60GHz Test 10 threads read json to map: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------ Test use multi mapper 1193 2305 1573 0.8 1193.0 1.0X Test use singleton mapper 1048 1487 621 1.0 1047.6 1.1X OpenJDK 64-Bit Server VM 1.8.0_312-b07 on Linux 4.14.0_1-0-0-45 Intel(R) Xeon(R) Gold 6271C CPU @ 2.60GHz Test 20 threads read json to map: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------ Test use multi mapper 1113 1491 535 0.9 1113.3 1.0X Test use singleton mapper 1351 1455 147 0.7 1351.0 0.8X OpenJDK 64-Bit Server VM 1.8.0_312-b07 on Linux 4.14.0_1-0-0-45 Intel(R) Xeon(R) Gold 6271C CPU @ 2.60GHz Test 50 threads read json to map: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------ Test use multi mapper 1003 1471 661 1.0 1003.3 1.0X Test use singleton mapper 1000 1330 468 1.0 999.8 1.0X OpenJDK 64-Bit Server VM 1.8.0_312-b07 on Linux 4.14.0_1-0-0-45 Intel(R) Xeon(R) Gold 6271C CPU @ 2.60GHz Test 75 threads read json to map: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------ Test use multi mapper 1019 1345 461 1.0 1018.7 1.0X Test use singleton mapper 1024 1241 306 1.0 1024.3 1.0X OpenJDK 64-Bit Server VM 1.8.0_312-b07 on Linux 4.14.0_1-0-0-45 Intel(R) Xeon(R) Gold 6271C CPU @ 2.60GHz Test 5 threads write map to json: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------ Test use multi mapper 842 1474 894 1.2 842.0 1.0X Test use singleton mapper 959 1585 884 1.0 959.5 0.9X OpenJDK 64-Bit Server VM 1.8.0_312-b07 on Linux 4.14.0_1-0-0-45 Intel(R) Xeon(R) Gold 6271C CPU @ 2.60GHz Test 10 threads write map to json: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------ Test use multi mapper 1280 1844 797 0.8 1280.2 1.0X Test use singleton mapper 889 1634 1054 1.1 888.5 1.4X OpenJDK 64-Bit Server VM 1.8.0_312-b07 on Linux 4.14.0_1-0-0-45 Intel(R) Xeon(R) Gold 6271C CPU @ 2.60GHz Test 20 threads write map to json: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------ Test use multi mapper 858 957 132 1.2 857.6 1.0X Test use singleton mapper 896 1625 1031 1.1 895.6 1.0X OpenJDK 64-Bit Server VM 1.8.0_312-b07 on Linux 4.14.0_1-0-0-45 Intel(R) Xeon(R) Gold 6271C CPU @ 2.60GHz Test 50 threads write map to json: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------ Test use multi mapper 884 960 115 1.1 884.2 1.0X Test use singleton mapper 843 1227 626 1.2 843.3 1.0X OpenJDK 64-Bit Server VM 1.8.0_312-b07 on Linux 4.14.0_1-0-0-45 Intel(R) Xeon(R) Gold 6271C CPU @ 2.60GHz Test 75 threads write map to json: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------ Test use multi mapper 1094 1109 22 0.9 1093.7 1.0X Test use singleton mapper 872 1257 658 1.1 871.5 1.3X ``` -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
