[
https://issues.apache.org/jira/browse/HADOOP-10674?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=14029766#comment-14029766
]
Tsz Wo Nicholas Sze commented on HADOOP-10674:
----------------------------------------------
- For Java 7, the new code is also better than the existing code but
java.util.zip.CRC32 remains the best.
$JAVA7_HOME/bin/java -cp target/classes/:target/test-classes/
'org.apache.hadoop.util.TestPureJavaCrc32$PerformanceTest'
java.version = 1.7.0_60
java.runtime.name = Java(TM) SE Runtime Environment
java.runtime.version = 1.7.0_60-b19
java.vm.version = 24.60-b09
java.vm.vendor = Oracle Corporation
java.vm.name = Java HotSpot(TM) 64-Bit Server VM
java.vm.specification.version = 1.7
java.specification.version = 1.7
os.arch = x86_64
os.name = Mac OS X
os.version = 10.9.3
Performance Table (The unit is MB/sec; #T = #Theads)
| #Bytes | #T || CRC32 || PureJavaCrc32 | % diff || PureJavaCrc32new | %
diff | % diff |
| 32 | 1 | 289.5 | 1224.5 | 323.0% | 1183.9 |
309.0% | -3.3% |
| 32 | 2 | 288.3 | 1224.2 | 324.6% | 1195.1 |
314.5% | -2.4% |
| 32 | 4 | 239.4 | 990.1 | 313.6% | 977.2 |
308.2% | -1.3% |
| 32 | 8 | 189.5 | 648.7 | 242.3% | 644.8 |
240.2% | -0.6% |
| 32 | 16 | 98.0 | 375.2 | 282.9% | 376.2 |
283.9% | 0.3% |
| #Bytes | #T || CRC32 || PureJavaCrc32 | % diff || PureJavaCrc32new | %
diff | % diff |
| 64 | 1 | 515.6 | 1433.6 | 178.0% | 1497.1 |
190.4% | 4.4% |
| 64 | 2 | 517.2 | 1401.3 | 170.9% | 1471.5 |
184.5% | 5.0% |
| 64 | 4 | 404.3 | 1155.3 | 185.7% | 1209.6 |
199.2% | 4.7% |
| 64 | 8 | 343.2 | 756.6 | 120.4% | 767.7 |
123.7% | 1.5% |
| 64 | 16 | 185.3 | 444.7 | 140.0% | 468.6 |
153.0% | 5.4% |
| #Bytes | #T || CRC32 || PureJavaCrc32 | % diff || PureJavaCrc32new | %
diff | % diff |
| 128 | 1 | 918.8 | 1539.7 | 67.6% | 1644.3 |
79.0% | 6.8% |
| 128 | 2 | 859.7 | 1503.8 | 74.9% | 1626.6 |
89.2% | 8.2% |
| 128 | 4 | 763.6 | 1339.2 | 75.4% | 1328.5 |
74.0% | -0.8% |
| 128 | 8 | 586.7 | 791.8 | 34.9% | 863.0 |
47.1% | 9.0% |
| 128 | 16 | 331.2 | 486.5 | 46.9% | 516.5 |
56.0% | 6.2% |
| #Bytes | #T || CRC32 || PureJavaCrc32 | % diff || PureJavaCrc32new | %
diff | % diff |
| 256 | 1 | 1410.1 | 1396.4 | -1.0% | 1678.4 |
19.0% | 20.2% |
| 256 | 2 | 1290.1 | 1492.0 | 15.7% | 1643.5 |
27.4% | 10.2% |
| 256 | 4 | 1174.9 | 1259.8 | 7.2% | 1383.5 |
17.8% | 9.8% |
| 256 | 8 | 859.2 | 846.0 | -1.5% | 905.2 |
5.4% | 7.0% |
| 256 | 16 | 512.1 | 508.4 | -0.7% | 551.5 |
7.7% | 8.5% |
| #Bytes | #T || CRC32 || PureJavaCrc32 | % diff || PureJavaCrc32new | %
diff | % diff |
| 512 | 1 | 1921.1 | 1585.7 | -17.5% | 1767.7 |
-8.0% | 11.5% |
| 512 | 2 | 1906.7 | 1565.0 | -17.9% | 1740.8 |
-8.7% | 11.2% |
| 512 | 4 | 1537.8 | 1461.9 | -4.9% | 1510.8 |
-1.8% | 3.3% |
| 512 | 8 | 1170.8 | 844.7 | -27.8% | 890.5 |
-23.9% | 5.4% |
| 512 | 16 | 738.1 | 522.5 | -29.2% | 576.7 |
-21.9% | 10.4% |
| #Bytes | #T || CRC32 || PureJavaCrc32 | % diff || PureJavaCrc32new | %
diff | % diff |
| 1024 | 1 | 2354.0 | 1584.7 | -32.7% | 1798.5 |
-23.6% | 13.5% |
| 1024 | 2 | 2245.7 | 1586.6 | -29.3% | 1780.7 |
-20.7% | 12.2% |
| 1024 | 4 | 2002.8 | 1322.5 | -34.0% | 1362.8 |
-32.0% | 3.0% |
| 1024 | 8 | 1423.3 | 890.6 | -37.4% | 956.3 |
-32.8% | 7.4% |
| 1024 | 16 | 997.4 | 530.4 | -46.8% | 673.9 |
-32.4% | 27.1% |
| #Bytes | #T || CRC32 || PureJavaCrc32 | % diff || PureJavaCrc32new | %
diff | % diff |
| 2048 | 1 | 2631.5 | 1577.7 | -40.0% | 1821.2 |
-30.8% | 15.4% |
| 2048 | 2 | 2577.6 | 1614.9 | -37.3% | 1802.7 |
-30.1% | 11.6% |
| 2048 | 4 | 2212.4 | 1375.5 | -37.8% | 1468.3 |
-33.6% | 6.7% |
| 2048 | 8 | 1581.4 | 892.1 | -43.6% | 964.9 |
-39.0% | 8.2% |
| 2048 | 16 | 1288.3 | 538.6 | -58.2% | 578.2 |
-55.1% | 7.4% |
| #Bytes | #T || CRC32 || PureJavaCrc32 | % diff || PureJavaCrc32new | %
diff | % diff |
| 4096 | 1 | 2754.0 | 1635.7 | -40.6% | 1844.2 |
-33.0% | 12.7% |
| 4096 | 2 | 2731.3 | 1617.1 | -40.8% | 1815.6 |
-33.5% | 12.3% |
| 4096 | 4 | 2339.7 | 1362.2 | -41.8% | 1615.8 |
-30.9% | 18.6% |
| 4096 | 8 | 1687.1 | 897.6 | -46.8% | 811.2 |
-51.9% | -9.6% |
| 4096 | 16 | 1265.4 | 535.4 | -57.7% | 588.2 |
-53.5% | 9.9% |
| #Bytes | #T || CRC32 || PureJavaCrc32 | % diff || PureJavaCrc32new | %
diff | % diff |
| 8192 | 1 | 2826.2 | 1631.7 | -42.3% | 1756.1 |
-37.9% | 7.6% |
| 8192 | 2 | 2815.2 | 1593.6 | -43.4% | 1714.9 |
-39.1% | 7.6% |
| 8192 | 4 | 2413.9 | 1365.2 | -43.4% | 1493.6 |
-38.1% | 9.4% |
| 8192 | 8 | 1727.5 | 898.4 | -48.0% | 975.6 |
-43.5% | 8.6% |
| 8192 | 16 | 1335.2 | 461.9 | -65.4% | 596.3 |
-55.3% | 29.1% |
| #Bytes | #T || CRC32 || PureJavaCrc32 | % diff || PureJavaCrc32new | %
diff | % diff |
| 16384 | 1 | 2900.8 | 1656.0 | -42.9% | 1846.9 |
-36.3% | 11.5% |
| 16384 | 2 | 2881.9 | 1627.8 | -43.5% | 1829.6 |
-36.5% | 12.4% |
| 16384 | 4 | 2438.8 | 1352.4 | -44.5% | 1650.8 |
-32.3% | 22.1% |
| 16384 | 8 | 1762.1 | 904.9 | -48.6% | 973.3 |
-44.8% | 7.6% |
| 16384 | 16 | 1375.1 | 539.2 | -60.8% | 605.3 |
-56.0% | 12.3% |
| #Bytes | #T || CRC32 || PureJavaCrc32 | % diff || PureJavaCrc32new | %
diff | % diff |
| 32768 | 1 | 2919.8 | 1651.9 | -43.4% | 1838.9 |
-37.0% | 11.3% |
| 32768 | 2 | 2888.1 | 1622.1 | -43.8% | 1804.3 |
-37.5% | 11.2% |
| 32768 | 4 | 2403.6 | 1364.2 | -43.2% | 1498.8 |
-37.6% | 9.9% |
| 32768 | 8 | 1772.0 | 904.5 | -49.0% | 978.4 |
-44.8% | 8.2% |
| 32768 | 16 | 1397.8 | 544.4 | -61.1% | 593.1 |
-57.6% | 8.9% |
| #Bytes | #T || CRC32 || PureJavaCrc32 | % diff || PureJavaCrc32new | %
diff | % diff |
| 65536 | 1 | 2940.1 | 1641.9 | -44.2% | 1854.6 |
-36.9% | 13.0% |
| 65536 | 2 | 2896.4 | 1621.5 | -44.0% | 1710.8 |
-40.9% | 5.5% |
| 65536 | 4 | 2523.5 | 1342.2 | -46.8% | 1493.8 |
-40.8% | 11.3% |
| 65536 | 8 | 1788.8 | 898.0 | -49.8% | 976.5 |
-45.4% | 8.7% |
| 65536 | 16 | 1353.8 | 618.7 | -54.3% | 598.6 |
-55.8% | -3.3% |
| #Bytes | #T || CRC32 || PureJavaCrc32 | % diff || PureJavaCrc32new | %
diff | % diff |
| 131072 | 1 | 2966.6 | 1652.9 | -44.3% | 1856.8 |
-37.4% | 12.3% |
| 131072 | 2 | 2757.2 | 1631.7 | -40.8% | 1742.8 |
-36.8% | 6.8% |
| 131072 | 4 | 2489.1 | 1359.9 | -45.4% | 1502.5 |
-39.6% | 10.5% |
| 131072 | 8 | 1715.8 | 879.6 | -48.7% | 977.0 |
-43.1% | 11.1% |
| 131072 | 16 | 1446.4 | 590.7 | -59.2% | 602.5 |
-58.3% | 2.0% |
| #Bytes | #T || CRC32 || PureJavaCrc32 | % diff || PureJavaCrc32new | %
diff | % diff |
| 262144 | 1 | 2662.5 | 1643.4 | -38.3% | 1846.7 |
-30.6% | 12.4% |
| 262144 | 2 | 2918.2 | 1625.7 | -44.3% | 1818.0 |
-37.7% | 11.8% |
| 262144 | 4 | 2449.4 | 1367.4 | -44.2% | 1497.1 |
-38.9% | 9.5% |
| 262144 | 8 | 1771.2 | 902.7 | -49.0% | 973.9 |
-45.0% | 7.9% |
| 262144 | 16 | 1402.8 | 551.6 | -60.7% | 600.3 |
-57.2% | 8.8% |
| #Bytes | #T || CRC32 || PureJavaCrc32 | % diff || PureJavaCrc32new | %
diff | % diff |
| 524288 | 1 | 2915.3 | 1638.8 | -43.8% | 1846.2 |
-36.7% | 12.7% |
| 524288 | 2 | 2930.4 | 1621.9 | -44.7% | 1753.5 |
-40.2% | 8.1% |
| 524288 | 4 | 2438.1 | 1332.8 | -45.3% | 1644.7 |
-32.5% | 23.4% |
| 524288 | 8 | 1779.5 | 900.9 | -49.4% | 975.2 |
-45.2% | 8.2% |
| 524288 | 16 | 1350.5 | 546.3 | -59.5% | 606.0 |
-55.1% | 10.9% |
| #Bytes | #T || CRC32 || PureJavaCrc32 | % diff || PureJavaCrc32new | %
diff | % diff |
| 1048576 | 1 | 2956.9 | 1654.9 | -44.0% | 1858.7 |
-37.1% | 12.3% |
| 1048576 | 2 | 2914.6 | 1631.0 | -44.0% | 1830.1 |
-37.2% | 12.2% |
| 1048576 | 4 | 2626.5 | 1366.1 | -48.0% | 1491.5 |
-43.2% | 9.2% |
| 1048576 | 8 | 1788.6 | 899.6 | -49.7% | 975.0 |
-45.5% | 8.4% |
| 1048576 | 16 | 1411.0 | 560.2 | -60.3% | 598.2 |
-57.6% | 6.8% |
| #Bytes | #T || CRC32 || PureJavaCrc32 | % diff || PureJavaCrc32new | %
diff | % diff |
| 2097152 | 1 | 2939.2 | 1637.0 | -44.3% | 1731.5 |
-41.1% | 5.8% |
| 2097152 | 2 | 2935.1 | 1620.7 | -44.8% | 1809.5 |
-38.4% | 11.7% |
| 2097152 | 4 | 2814.6 | 1362.0 | -51.6% | 1546.0 |
-45.1% | 13.5% |
| 2097152 | 8 | 1758.8 | 895.7 | -49.1% | 970.5 |
-44.8% | 8.4% |
| 2097152 | 16 | 1415.4 | 541.5 | -61.7% | 598.0 |
-57.7% | 10.4% |
| #Bytes | #T || CRC32 || PureJavaCrc32 | % diff || PureJavaCrc32new | %
diff | % diff |
| 4194304 | 1 | 2926.1 | 1643.8 | -43.8% | 1841.4 |
-37.1% | 12.0% |
| 4194304 | 2 | 2923.8 | 1618.1 | -44.7% | 1811.9 |
-38.0% | 12.0% |
| 4194304 | 4 | 2603.4 | 1289.1 | -50.5% | 1550.2 |
-40.5% | 20.2% |
| 4194304 | 8 | 1780.0 | 881.3 | -50.5% | 951.2 |
-46.6% | 7.9% |
| 4194304 | 16 | 1470.4 | 535.2 | -63.6% | 599.1 |
-59.3% | 11.9% |
| #Bytes | #T || CRC32 || PureJavaCrc32 | % diff || PureJavaCrc32new | %
diff | % diff |
| 8388608 | 1 | 2958.0 | 1608.5 | -45.6% | 1846.3 |
-37.6% | 14.8% |
| 8388608 | 2 | 2906.1 | 1614.5 | -44.4% | 1808.3 |
-37.8% | 12.0% |
| 8388608 | 4 | 2461.7 | 1316.5 | -46.5% | 1510.9 |
-38.6% | 14.8% |
| 8388608 | 8 | 1769.9 | 896.5 | -49.3% | 954.1 |
-46.1% | 6.4% |
| 8388608 | 16 | 1430.4 | 560.3 | -60.8% | 584.8 |
-59.1% | 4.4% |
| #Bytes | #T || CRC32 || PureJavaCrc32 | % diff || PureJavaCrc32new | %
diff | % diff |
| 16777216 | 1 | 2960.9 | 1631.9 | -44.9% | 1842.4 |
-37.8% | 12.9% |
| 16777216 | 2 | 2903.7 | 1622.5 | -44.1% | 1795.5 |
-38.2% | 10.7% |
| 16777216 | 4 | 2349.7 | 1329.6 | -43.4% | 1492.2 |
-36.5% | 12.2% |
| 16777216 | 8 | 1715.8 | 898.4 | -47.6% | 966.8 |
-43.7% | 7.6% |
| 16777216 | 16 | 1385.0 | 539.0 | -61.1% | 587.5 |
-57.6% | 9.0% |
| #Bytes | #T || CRC32 || PureJavaCrc32 | % diff || PureJavaCrc32new | %
diff | % diff |
| 33554432 | 1 | 2974.9 | 1600.0 | -46.2% | 1833.1 |
-38.4% | 14.6% |
| 33554432 | 2 | 2906.4 | 1609.7 | -44.6% | 1802.0 |
-38.0% | 11.9% |
| 33554432 | 4 | 2574.8 | 1359.3 | -47.2% | 1489.7 |
-42.1% | 9.6% |
| 33554432 | 8 | 1763.1 | 807.6 | -54.2% | 968.0 |
-45.1% | 19.9% |
| 33554432 | 16 | 1460.5 | 547.4 | -62.5% | 598.5 |
-59.0% | 9.3% |
> Rewrite the PureJavaCrc32 loop for performance improvement
> ----------------------------------------------------------
>
> Key: HADOOP-10674
> URL: https://issues.apache.org/jira/browse/HADOOP-10674
> Project: Hadoop Common
> Issue Type: Improvement
> Components: performance, util
> Reporter: Tsz Wo Nicholas Sze
> Assignee: Tsz Wo Nicholas Sze
> Attachments: c10674_20140609.patch, c10674_20140609b.patch,
> c10674_20140610.patch
>
>
> Below are some performance improvement opportunities performance improvement
> in PureJavaCrc32.
> - eliminate "off += 8; len -= 8;"
> - replace T8_x_start with hard coded constants
> - eliminate c0 - c7 local variables
> In my machine, there are 30% to 50% improvement for most of the cases.
--
This message was sent by Atlassian JIRA
(v6.2#6252)
