[
https://issues.apache.org/jira/browse/HADOOP-15033?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16256717#comment-16256717
]
Dmitry Chuyko commented on HADOOP-15033:
----------------------------------------
Results on latest JDK 10 on 4 core x86 including native library:
java.version = 10-internal
java.runtime.name = OpenJDK Runtime Environment
java.runtime.version = 10-internal+0-adhoc.tp.hs10
java.vm.version = 10-internal+0-adhoc.tp.hs10
java.vm.vendor = Oracle Corporation
java.vm.name = OpenJDK 64-Bit Server VM
java.vm.specification.version = 10
java.specification.version = 10
os.arch = amd64
os.name = Linux
os.version = 4.13.0-16-generic
Data Length = 64 MB
Trials = 5
Direct Buffer Performance Table (bpc: byte-per-crc in MB/sec; #T: #Theads)
| bpc | #T || Zip || ZipC | % diff || PureJava | % diff || PureJavaC
| % diff || Native | % diff || NativeC | % diff |
| 32 | 1 | 540.5 | 921.7 | 70.5% | 644.0 | -30.1% | 662.5
| 2.9% | 1535.3 | 131.7% | 8598.1 | 460.0% |
| 32 | 2 | 346.0 | 489.6 | 41.5% | 325.3 | -33.6% | 303.2
| -6.8% | 898.0 | 196.2% | 4720.6 | 425.7% |
| 32 | 4 | 226.8 | 384.1 | 69.3% | 238.6 | -37.9% | 242.5
| 1.6% | 731.9 | 201.9% | 3838.0 | 424.4% |
| 32 | 8 | 104.1 | 213.6 | 105.1% | 147.6 | -30.9% | 141.9
| -3.9% | 386.5 | 172.4% | 2283.9 | 490.9% |
| 32 | 16 | 70.0 | 110.1 | 57.1% | 77.6 | -29.5% | 74.9
| -3.5% | 223.6 | 198.6% | 1657.4 | 641.1% |
| bpc | #T || Zip || ZipC | % diff || PureJava | % diff || PureJavaC
| % diff || Native | % diff || NativeC | % diff |
| 64 | 1 | 828.0 | 1266.5 | 53.0% | 876.1 | -30.8% | 833.9
| -4.8% | 1592.7 | 91.0% | 10217.4 | 541.5% |
| 64 | 2 | 549.2 | 753.1 | 37.1% | 420.2 | -44.2% | 438.2
| 4.3% | 1070.3 | 144.3% | 6534.0 | 510.5% |
| 64 | 4 | 384.0 | 627.6 | 63.5% | 348.8 | -44.4% | 365.9
| 4.9% | 854.0 | 133.4% | 4762.2 | 457.6% |
| 64 | 8 | 226.6 | 317.2 | 40.0% | 195.5 | -38.4% | 218.6
| 11.8% | 488.6 | 123.5% | 2760.0 | 464.9% |
| 64 | 16 | 113.7 | 176.1 | 54.9% | 113.8 | -35.4% | 125.4
| 10.2% | 255.9 | 104.1% | 2556.6 | 899.1% |
| bpc | #T || Zip || ZipC | % diff || PureJava | % diff || PureJavaC
| % diff || Native | % diff || NativeC | % diff |
| 128 | 1 | 1146.1 | 2241.7 | 95.6% | 1080.8 | -51.8% | 1008.1
| -6.7% | 1626.2 | 61.3% | 11525.3 | 608.7% |
| 128 | 2 | 792.6 | 1248.3 | 57.5% | 536.0 | -57.1% | 786.1
| 46.7% | 1360.0 | 73.0% | 7295.6 | 436.5% |
| 128 | 4 | 602.7 | 1074.4 | 78.3% | 456.8 | -57.5% | 437.4
| -4.2% | 927.7 | 112.1% | 6405.6 | 590.5% |
| 128 | 8 | 327.6 | 607.0 | 85.3% | 265.6 | -56.2% | 245.3
| -7.6% | 499.6 | 103.7% | 3545.9 | 609.8% |
| 128 | 16 | 176.2 | 295.0 | 67.5% | 150.7 | -48.9% | 132.4
| -12.1% | 287.0 | 116.7% | 2865.1 | 898.3% |
| bpc | #T || Zip || ZipC | % diff || PureJava | % diff || PureJavaC
| % diff || Native | % diff || NativeC | % diff |
| 256 | 1 | 1647.9 | 2802.3 | 70.0% | 1240.3 | -55.7% | 1090.2
| -12.1% | 1525.0 | 39.9% | 11362.2 | 645.0% |
| 256 | 2 | 1021.1 | 1621.4 | 58.8% | 642.2 | -60.4% | 616.8
| -4.0% | 1241.3 | 101.3% | 7501.2 | 504.3% |
| 256 | 4 | 879.3 | 1368.5 | 55.6% | 494.9 | -63.8% | 497.2
| 0.5% | 1001.9 | 101.5% | 5424.7 | 441.4% |
| 256 | 8 | 448.7 | 705.8 | 57.3% | 302.1 | -57.2% | 266.1
| -11.9% | 529.9 | 99.1% | 4327.7 | 716.7% |
| 256 | 16 | 245.4 | 477.3 | 94.5% | 147.3 | -69.1% | 152.9
| 3.8% | 325.5 | 113.0% | 2456.4 | 654.6% |
| bpc | #T || Zip || ZipC | % diff || PureJava | % diff || PureJavaC
| % diff || Native | % diff || NativeC | % diff |
| 512 | 1 | 1872.1 | 2954.9 | 57.8% | 1307.1 | -55.8% | 1102.0
| -15.7% | 1586.3 | 44.0% | 9733.8 | 513.6% |
| 512 | 2 | 1237.1 | 1940.5 | 56.9% | 707.8 | -63.5% | 664.8
| -6.1% | 1278.6 | 92.3% | 7194.6 | 462.7% |
| 512 | 4 | 997.9 | 1619.2 | 62.3% | 610.2 | -62.3% | 553.2
| -9.4% | 1035.6 | 87.2% | 5029.4 | 385.7% |
| 512 | 8 | 554.1 | 974.3 | 75.8% | 329.6 | -66.2% | 299.1
| -9.3% | 596.8 | 99.5% | 4345.3 | 628.1% |
| 512 | 16 | 327.0 | 578.3 | 76.9% | 186.9 | -67.7% | 178.8
| -4.4% | 336.4 | 88.2% | 3274.8 | 873.4% |
| bpc | #T || Zip || ZipC | % diff || PureJava | % diff || PureJavaC
| % diff || Native | % diff || NativeC | % diff |
| 1024 | 1 | 1898.0 | 4200.0 | 121.3% | 1269.3 | -69.8% | 1139.9
| -10.2% | 1594.5 | 39.9% | 8967.9 | 462.4% |
| 1024 | 2 | 1318.7 | 3191.4 | 142.0% | 735.3 | -77.0% | 681.7
| -7.3% | 1270.1 | 86.3% | 6518.3 | 413.2% |
| 1024 | 4 | 1084.4 | 2557.6 | 135.9% | 592.4 | -76.8% | 464.1
| -21.7% | 1020.1 | 119.8% | 4907.7 | 381.1% |
| 1024 | 8 | 569.0 | 1536.8 | 170.1% | 340.5 | -77.8% | 319.6
| -6.1% | 618.7 | 93.6% | 3556.5 | 474.8% |
| 1024 | 16 | 373.3 | 1075.6 | 188.1% | 211.0 | -80.4% | 173.1
| -18.0% | 316.9 | 83.1% | 3075.3 | 870.5% |
| bpc | #T || Zip || ZipC | % diff || PureJava | % diff || PureJavaC
| % diff || Native | % diff || NativeC | % diff |
| 2048 | 1 | 1941.4 | 6284.6 | 223.7% | 1177.0 | -81.3% | 1117.9
| -5.0% | 1567.7 | 40.2% | 11286.2 | 619.9% |
| 2048 | 2 | 1372.7 | 4693.6 | 241.9% | 752.3 | -84.0% | 697.5
| -7.3% | 1284.3 | 84.1% | 6847.4 | 433.1% |
| 2048 | 4 | 1154.3 | 3973.6 | 244.3% | 610.1 | -84.6% | 579.9
| -5.0% | 1040.5 | 79.4% | 5185.0 | 398.3% |
| 2048 | 8 | 682.8 | 2454.2 | 259.5% | 317.5 | -87.1% | 294.9
| -7.1% | 523.8 | 77.6% | 3465.7 | 561.6% |
| 2048 | 16 | 374.2 | 1632.7 | 336.4% | 218.3 | -86.6% | 158.4
| -27.5% | 321.5 | 103.0% | 2287.9 | 611.6% |
| bpc | #T || Zip || ZipC | % diff || PureJava | % diff || PureJavaC
| % diff || Native | % diff || NativeC | % diff |
| 4096 | 1 | 2000.7 | 6184.3 | 209.1% | 1319.8 | -78.7% | 1153.9
| -12.6% | 1569.4 | 36.0% | 14192.6 | 804.3% |
| 4096 | 2 | 1406.0 | 4810.7 | 242.2% | 765.6 | -84.1% | 845.3
| 10.4% | 1372.4 | 62.4% | 7586.7 | 452.8% |
| 4096 | 4 | 1286.1 | 3817.8 | 196.9% | 591.7 | -84.5% | 577.7
| -2.4% | 1060.3 | 83.6% | 4602.5 | 334.1% |
| 4096 | 8 | 640.3 | 2468.6 | 285.5% | 411.6 | -83.3% | 367.4
| -10.7% | 717.8 | 95.4% | 4363.7 | 507.9% |
| 4096 | 16 | 392.5 | 1511.6 | 285.2% | 208.1 | -86.2% | 189.6
| -8.9% | 355.2 | 87.3% | 2104.0 | 492.4% |
| bpc | #T || Zip || ZipC | % diff || PureJava | % diff || PureJavaC
| % diff || Native | % diff || NativeC | % diff |
| 8192 | 1 | 2262.1 | 7183.6 | 217.6% | 1416.8 | -80.3% | 1237.3
| -12.7% | 1634.3 | 32.1% | 13987.3 | 755.9% |
| 8192 | 2 | 2101.0 | 6417.1 | 205.4% | 1308.4 | -79.6% | 1123.2
| -14.2% | 1564.0 | 39.2% | 9135.9 | 484.1% |
| 8192 | 4 | 1364.5 | 4136.8 | 203.2% | 745.8 | -82.0% | 696.1
| -6.7% | 1266.3 | 81.9% | 5036.4 | 297.7% |
| 8192 | 8 | 888.1 | 2815.2 | 217.0% | 452.8 | -83.9% | 366.8
| -19.0% | 703.0 | 91.7% | 3449.4 | 390.7% |
| 8192 | 16 | 415.2 | 1726.8 | 315.9% | 204.7 | -88.1% | 183.7
| -10.2% | 378.6 | 106.1% | 2575.1 | 580.1% |
| bpc | #T || Zip || ZipC | % diff || PureJava | % diff || PureJavaC
| % diff || Native | % diff || NativeC | % diff |
| 16384 | 1 | 2228.8 | 7937.4 | 256.1% | 1360.4 | -82.9% | 1193.6
| -12.3% | 1658.1 | 38.9% | 13390.5 | 707.6% |
| 16384 | 2 | 2079.8 | 6250.7 | 200.5% | 1188.1 | -81.0% | 1132.3
| -4.7% | 1557.4 | 37.5% | 9151.5 | 487.6% |
| 16384 | 4 | 1386.2 | 4004.0 | 188.8% | 753.8 | -81.2% | 693.2
| -8.0% | 1266.4 | 82.7% | 5427.0 | 328.5% |
| 16384 | 8 | 840.8 | 2602.3 | 209.5% | 406.4 | -84.4% | 369.6
| -9.1% | 712.6 | 92.8% | 2905.2 | 307.7% |
| 16384 | 16 | 411.7 | 1560.5 | 279.1% | 202.2 | -87.0% | 189.7
| -6.2% | 343.6 | 81.1% | 1929.8 | 461.6% |
| bpc | #T || Zip || ZipC | % diff || PureJava | % diff || PureJavaC
| % diff || Native | % diff || NativeC | % diff |
| 32768 | 1 | 2006.7 | 6119.1 | 204.9% | 1256.6 | -79.5% | 1119.9
| -10.9% | 1511.2 | 34.9% | 12599.6 | 733.7% |
| 32768 | 2 | 2028.2 | 6277.4 | 209.5% | 1270.2 | -79.8% | 1147.6
| -9.7% | 1569.7 | 36.8% | 9203.5 | 486.3% |
| 32768 | 4 | 1385.8 | 3964.2 | 186.1% | 757.0 | -80.9% | 690.8
| -8.7% | 1265.1 | 83.1% | 5282.2 | 317.5% |
| 32768 | 8 | 841.7 | 2716.0 | 222.7% | 449.3 | -83.5% | 360.9
| -19.7% | 759.7 | 110.5% | 2928.3 | 285.4% |
| 32768 | 16 | 421.1 | 1627.8 | 286.6% | 208.1 | -87.2% | 188.5
| -9.4% | 370.6 | 96.7% | 2658.2 | 617.2% |
| bpc | #T || Zip || ZipC | % diff || PureJava | % diff || PureJavaC
| % diff || Native | % diff || NativeC | % diff |
| 65536 | 1 | 2126.1 | 7479.8 | 251.8% | 1396.5 | -81.3% | 1218.2
| -12.8% | 1672.4 | 37.3% | 14690.7 | 778.4% |
| 65536 | 2 | 2074.1 | 6388.6 | 208.0% | 1330.3 | -79.2% | 1148.3
| -13.7% | 1578.4 | 37.5% | 9262.9 | 486.9% |
| 65536 | 4 | 1416.6 | 3926.3 | 177.2% | 758.9 | -80.7% | 701.4
| -7.6% | 1223.3 | 74.4% | 5554.2 | 354.0% |
| 65536 | 8 | 814.1 | 2571.6 | 215.9% | 410.3 | -84.0% | 367.2
| -10.5% | 753.4 | 105.2% | 5090.8 | 575.7% |
| 65536 | 16 | 488.5 | 1676.2 | 243.1% | 224.0 | -86.6% | 198.8
| -11.3% | 355.3 | 78.8% | 2549.3 | 617.5% |
> Use java.util.zip.CRC32C for Java 9 and above
> ---------------------------------------------
>
> Key: HADOOP-15033
> URL: https://issues.apache.org/jira/browse/HADOOP-15033
> Project: Hadoop Common
> Issue Type: Improvement
> Components: performance, util
> Affects Versions: 3.0.0
> Reporter: Dmitry Chuyko
>
> java.util.zip.CRC32C implementation is available since Java 9.
> https://docs.oracle.com/javase/9/docs/api/java/util/zip/CRC32C.html
> Platform specific assembler intrinsics make it more effective than any pure
> Java implementation.
> Hadoop is compiled against Java 8 but class constructor may be accessible
> with method handle on 9 to instances implementing Checksum in runtime.
--
This message was sent by Atlassian JIRA
(v6.4.14#64029)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
