[
https://issues.apache.org/jira/browse/HADOOP-10674?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=14029753#comment-14029753
]
Tsz Wo Nicholas Sze commented on HADOOP-10674:
----------------------------------------------
- Multiple threaded tests. Then new code runs even better.
java.version = 1.6.0_65
java.runtime.name = Java(TM) SE Runtime Environment
java.runtime.version = 1.6.0_65-b14-462-11M4609
java.vm.version = 20.65-b04-462
java.vm.vendor = Apple Inc.
java.vm.name = Java HotSpot(TM) 64-Bit Server VM
java.vm.specification.version = 1.0
java.specification.version = 1.6
os.arch = x86_64
os.name = Mac OS X
os.version = 10.9.3
Performance Table (The unit is MB/sec; #T = #Theads)
| #Bytes | #T || CRC32 || PureJavaCrc32 | % diff || PureJavaCrc32new | %
diff | % diff |
| 32 | 1 | 255.4 | 945.8 | 270.3% | 1201.3 |
370.4% | 27.0% |
| 32 | 2 | 185.6 | 920.9 | 396.3% | 1360.5 |
633.2% | 47.7% |
| 32 | 4 | 124.4 | 738.2 | 493.2% | 966.2 |
676.4% | 30.9% |
| 32 | 8 | 78.4 | 463.0 | 490.9% | 644.2 |
722.1% | 39.1% |
| 32 | 16 | 40.8 | 253.8 | 522.7% | 354.9 |
770.8% | 39.8% |
| #Bytes | #T || CRC32 || PureJavaCrc32 | % diff || PureJavaCrc32new | %
diff | % diff |
| 64 | 1 | 304.6 | 1049.4 | 244.5% | 1439.6 |
372.6% | 37.2% |
| 64 | 2 | 264.1 | 1031.1 | 290.5% | 1395.1 |
428.3% | 35.3% |
| 64 | 4 | 211.7 | 845.7 | 299.5% | 1139.8 |
438.5% | 34.8% |
| 64 | 8 | 141.9 | 503.0 | 254.4% | 763.1 |
437.6% | 51.7% |
| 64 | 16 | 77.0 | 272.0 | 253.3% | 419.1 |
444.3% | 54.1% |
| #Bytes | #T || CRC32 || PureJavaCrc32 | % diff || PureJavaCrc32new | %
diff | % diff |
| 128 | 1 | 357.0 | 1098.7 | 207.7% | 1541.0 |
331.6% | 40.3% |
| 128 | 2 | 314.6 | 1091.1 | 246.8% | 1582.5 |
403.0% | 45.0% |
| 128 | 4 | 283.2 | 831.8 | 193.8% | 1391.7 |
391.5% | 67.3% |
| 128 | 8 | 229.1 | 525.2 | 129.2% | 846.2 |
269.3% | 61.1% |
| 128 | 16 | 123.5 | 292.0 | 136.4% | 466.5 |
277.6% | 59.7% |
| #Bytes | #T || CRC32 || PureJavaCrc32 | % diff || PureJavaCrc32new | %
diff | % diff |
| 256 | 1 | 388.6 | 1131.3 | 191.1% | 1577.5 |
306.0% | 39.4% |
| 256 | 2 | 362.4 | 1116.6 | 208.1% | 1445.5 |
298.9% | 29.5% |
| 256 | 4 | 338.4 | 933.3 | 175.8% | 1294.8 |
282.6% | 38.7% |
| 256 | 8 | 305.5 | 537.9 | 76.1% | 894.9 |
193.0% | 66.4% |
| 256 | 16 | 162.8 | 300.9 | 84.8% | 508.0 |
212.0% | 68.8% |
| #Bytes | #T || CRC32 || PureJavaCrc32 | % diff || PureJavaCrc32new | %
diff | % diff |
| 512 | 1 | 398.7 | 1153.0 | 189.2% | 1616.9 |
305.5% | 40.2% |
| 512 | 2 | 384.9 | 1143.5 | 197.1% | 1694.2 |
340.2% | 48.2% |
| 512 | 4 | 370.3 | 911.0 | 146.0% | 1507.6 |
307.1% | 65.5% |
| 512 | 8 | 344.7 | 543.1 | 57.5% | 920.8 |
167.1% | 69.5% |
| 512 | 16 | 179.3 | 366.5 | 104.4% | 532.5 |
197.0% | 45.3% |
| #Bytes | #T || CRC32 || PureJavaCrc32 | % diff || PureJavaCrc32new | %
diff | % diff |
| 1024 | 1 | 399.9 | 1152.3 | 188.1% | 1756.4 |
339.2% | 52.4% |
| 1024 | 2 | 395.1 | 1130.5 | 186.1% | 1753.7 |
343.9% | 55.1% |
| 1024 | 4 | 383.7 | 940.5 | 145.1% | 1405.3 |
266.2% | 49.4% |
| 1024 | 8 | 363.3 | 548.3 | 50.9% | 941.8 |
159.3% | 71.7% |
| 1024 | 16 | 196.8 | 290.8 | 47.8% | 525.8 |
167.2% | 80.8% |
| #Bytes | #T || CRC32 || PureJavaCrc32 | % diff || PureJavaCrc32new | %
diff | % diff |
| 2048 | 1 | 408.0 | 1171.5 | 187.2% | 1783.7 |
337.2% | 52.3% |
| 2048 | 2 | 400.8 | 1166.6 | 191.1% | 1784.0 |
345.1% | 52.9% |
| 2048 | 4 | 391.4 | 999.8 | 155.4% | 1577.4 |
303.0% | 57.8% |
| 2048 | 8 | 362.1 | 546.2 | 50.8% | 910.6 |
151.5% | 66.7% |
| 2048 | 16 | 198.1 | 303.4 | 53.1% | 518.5 |
161.7% | 70.9% |
| #Bytes | #T || CRC32 || PureJavaCrc32 | % diff || PureJavaCrc32new | %
diff | % diff |
| 4096 | 1 | 406.7 | 1171.8 | 188.1% | 1773.7 |
336.1% | 51.4% |
| 4096 | 2 | 405.3 | 1104.5 | 172.5% | 1770.0 |
336.7% | 60.2% |
| 4096 | 4 | 395.9 | 975.9 | 146.5% | 1352.7 |
241.6% | 38.6% |
| 4096 | 8 | 364.8 | 535.9 | 46.9% | 924.2 |
153.4% | 72.5% |
| 4096 | 16 | 186.5 | 285.5 | 53.0% | 551.3 |
195.6% | 93.1% |
| #Bytes | #T || CRC32 || PureJavaCrc32 | % diff || PureJavaCrc32new | %
diff | % diff |
| 8192 | 1 | 411.1 | 1172.4 | 185.2% | 1807.7 |
339.7% | 54.2% |
| 8192 | 2 | 407.0 | 1168.9 | 187.2% | 1684.1 |
313.8% | 44.1% |
| 8192 | 4 | 399.5 | 940.5 | 135.4% | 1568.1 |
292.5% | 66.7% |
| 8192 | 8 | 381.7 | 552.7 | 44.8% | 959.6 |
151.4% | 73.6% |
| 8192 | 16 | 204.2 | 300.1 | 47.0% | 562.4 |
175.4% | 87.4% |
| #Bytes | #T || CRC32 || PureJavaCrc32 | % diff || PureJavaCrc32new | %
diff | % diff |
| 16384 | 1 | 414.1 | 1191.6 | 187.8% | 1832.1 |
342.5% | 53.8% |
| 16384 | 2 | 408.0 | 1166.7 | 186.0% | 1813.4 |
344.5% | 55.4% |
| 16384 | 4 | 399.3 | 956.9 | 139.6% | 1321.2 |
230.8% | 38.1% |
| 16384 | 8 | 382.2 | 551.2 | 44.2% | 963.0 |
152.0% | 74.7% |
| 16384 | 16 | 201.4 | 304.0 | 51.0% | 662.2 |
228.8% | 117.8% |
| #Bytes | #T || CRC32 || PureJavaCrc32 | % diff || PureJavaCrc32new | %
diff | % diff |
| 32768 | 1 | 407.1 | 1184.9 | 191.1% | 1834.7 |
350.7% | 54.8% |
| 32768 | 2 | 408.8 | 1174.7 | 187.3% | 1790.4 |
337.9% | 52.4% |
| 32768 | 4 | 400.7 | 878.9 | 119.3% | 1477.5 |
268.7% | 68.1% |
| 32768 | 8 | 372.2 | 548.9 | 47.5% | 916.1 |
146.2% | 66.9% |
| 32768 | 16 | 213.5 | 298.9 | 40.0% | 557.5 |
161.1% | 86.5% |
| #Bytes | #T || CRC32 || PureJavaCrc32 | % diff || PureJavaCrc32new | %
diff | % diff |
| 65536 | 1 | 410.9 | 1149.0 | 179.6% | 1744.6 |
324.6% | 51.8% |
| 65536 | 2 | 409.5 | 1171.7 | 186.1% | 1802.9 |
340.3% | 53.9% |
| 65536 | 4 | 401.3 | 1051.0 | 161.9% | 1440.6 |
258.9% | 37.1% |
| 65536 | 8 | 383.3 | 551.7 | 43.9% | 959.1 |
150.2% | 73.8% |
| 65536 | 16 | 211.8 | 336.4 | 58.9% | 554.2 |
161.7% | 64.7% |
| #Bytes | #T || CRC32 || PureJavaCrc32 | % diff || PureJavaCrc32new | %
diff | % diff |
| 131072 | 1 | 413.3 | 1183.3 | 186.3% | 1822.7 |
341.0% | 54.0% |
| 131072 | 2 | 409.8 | 1124.4 | 174.4% | 1800.1 |
339.3% | 60.1% |
| 131072 | 4 | 401.5 | 932.9 | 132.4% | 1566.2 |
290.1% | 67.9% |
| 131072 | 8 | 387.4 | 551.3 | 42.3% | 958.9 |
147.6% | 73.9% |
| 131072 | 16 | 226.3 | 297.2 | 31.4% | 542.5 |
139.7% | 82.5% |
| #Bytes | #T || CRC32 || PureJavaCrc32 | % diff || PureJavaCrc32new | %
diff | % diff |
| 262144 | 1 | 397.1 | 1192.4 | 200.3% | 1764.9 |
344.4% | 48.0% |
| 262144 | 2 | 410.4 | 1184.7 | 188.7% | 1811.2 |
341.3% | 52.9% |
| 262144 | 4 | 401.1 | 952.1 | 137.4% | 1593.1 |
297.1% | 67.3% |
| 262144 | 8 | 386.3 | 553.6 | 43.3% | 925.8 |
139.6% | 67.2% |
| 262144 | 16 | 211.4 | 299.7 | 41.8% | 559.6 |
164.7% | 86.7% |
| #Bytes | #T || CRC32 || PureJavaCrc32 | % diff || PureJavaCrc32new | %
diff | % diff |
| 524288 | 1 | 408.9 | 1193.3 | 191.8% | 1769.6 |
332.8% | 48.3% |
| 524288 | 2 | 407.6 | 1123.5 | 175.6% | 1804.9 |
342.8% | 60.7% |
| 524288 | 4 | 403.1 | 903.6 | 124.2% | 1430.7 |
255.0% | 58.3% |
| 524288 | 8 | 386.2 | 552.6 | 43.1% | 959.0 |
148.3% | 73.6% |
| 524288 | 16 | 203.8 | 307.4 | 50.8% | 576.7 |
182.9% | 87.6% |
| #Bytes | #T || CRC32 || PureJavaCrc32 | % diff || PureJavaCrc32new | %
diff | % diff |
| 1048576 | 1 | 408.5 | 1186.7 | 190.5% | 1824.5 |
346.6% | 53.7% |
| 1048576 | 2 | 409.9 | 1122.2 | 173.8% | 1799.6 |
339.0% | 60.4% |
| 1048576 | 4 | 402.0 | 962.5 | 139.4% | 1311.5 |
226.3% | 36.3% |
| 1048576 | 8 | 386.8 | 547.9 | 41.7% | 959.3 |
148.0% | 75.1% |
| 1048576 | 16 | 220.2 | 300.1 | 36.3% | 529.2 |
140.3% | 76.3% |
| #Bytes | #T || CRC32 || PureJavaCrc32 | % diff || PureJavaCrc32new | %
diff | % diff |
| 2097152 | 1 | 408.3 | 1177.7 | 188.4% | 1802.0 |
341.3% | 53.0% |
| 2097152 | 2 | 409.4 | 1108.9 | 170.9% | 1693.3 |
313.6% | 52.7% |
| 2097152 | 4 | 402.0 | 1015.7 | 152.6% | 1350.8 |
236.0% | 33.0% |
| 2097152 | 8 | 386.5 | 545.6 | 41.2% | 949.5 |
145.7% | 74.0% |
| 2097152 | 16 | 215.9 | 298.8 | 38.4% | 542.1 |
151.1% | 81.4% |
| #Bytes | #T || CRC32 || PureJavaCrc32 | % diff || PureJavaCrc32new | %
diff | % diff |
| 4194304 | 1 | 410.4 | 1177.4 | 186.9% | 1815.5 |
342.4% | 54.2% |
| 4194304 | 2 | 407.9 | 1173.4 | 187.7% | 1799.5 |
341.2% | 53.4% |
| 4194304 | 4 | 401.1 | 658.1 | 64.1% | 1586.4 |
295.5% | 141.0% |
| 4194304 | 8 | 377.7 | 548.3 | 45.2% | 955.6 |
153.0% | 74.3% |
| 4194304 | 16 | 212.2 | 300.2 | 41.4% | 542.1 |
155.4% | 80.6% |
| #Bytes | #T || CRC32 || PureJavaCrc32 | % diff || PureJavaCrc32new | %
diff | % diff |
| 8388608 | 1 | 409.9 | 1134.8 | 176.8% | 1797.5 |
338.5% | 58.4% |
| 8388608 | 2 | 404.0 | 1170.0 | 189.6% | 1741.1 |
331.0% | 48.8% |
| 8388608 | 4 | 401.7 | 889.8 | 121.5% | 1467.2 |
265.3% | 64.9% |
| 8388608 | 8 | 377.2 | 548.6 | 45.4% | 955.5 |
153.3% | 74.2% |
| 8388608 | 16 | 222.6 | 299.7 | 34.7% | 543.4 |
144.1% | 81.3% |
| #Bytes | #T || CRC32 || PureJavaCrc32 | % diff || PureJavaCrc32new | %
diff | % diff |
| 16777216 | 1 | 414.6 | 1161.3 | 180.1% | 1757.5 |
323.9% | 51.3% |
| 16777216 | 2 | 406.3 | 1119.7 | 175.5% | 1720.1 |
323.3% | 53.6% |
| 16777216 | 4 | 401.3 | 840.1 | 109.3% | 1361.5 |
239.3% | 62.1% |
| 16777216 | 8 | 382.4 | 555.8 | 45.3% | 950.9 |
148.7% | 71.1% |
| 16777216 | 16 | 218.7 | 303.5 | 38.8% | 570.5 |
160.9% | 88.0% |
| #Bytes | #T || CRC32 || PureJavaCrc32 | % diff || PureJavaCrc32new | %
diff | % diff |
| 33554432 | 1 | 413.7 | 1182.3 | 185.8% | 1765.1 |
326.7% | 49.3% |
| 33554432 | 2 | 410.5 | 1177.7 | 186.9% | 1666.5 |
305.9% | 41.5% |
| 33554432 | 4 | 400.0 | 857.4 | 114.3% | 1448.8 |
262.2% | 69.0% |
| 33554432 | 8 | 383.6 | 581.6 | 51.6% | 976.1 |
154.5% | 67.8% |
| 33554432 | 16 | 214.6 | 325.2 | 51.5% | 610.8 |
184.6% | 87.8% |
> Rewrite the PureJavaCrc32 loop for performance improvement
> ----------------------------------------------------------
>
> Key: HADOOP-10674
> URL: https://issues.apache.org/jira/browse/HADOOP-10674
> Project: Hadoop Common
> Issue Type: Improvement
> Components: performance, util
> Reporter: Tsz Wo Nicholas Sze
> Assignee: Tsz Wo Nicholas Sze
> Attachments: c10674_20140609.patch, c10674_20140609b.patch,
> c10674_20140610.patch
>
>
> Below are some performance improvement opportunities performance improvement
> in PureJavaCrc32.
> - eliminate "off += 8; len -= 8;"
> - replace T8_x_start with hard coded constants
> - eliminate c0 - c7 local variables
> In my machine, there are 30% to 50% improvement for most of the cases.
--
This message was sent by Atlassian JIRA
(v6.2#6252)
