Repository: cassandra Updated Branches: refs/heads/trunk 8fe1fdf5a -> 0d2ec11c7
use preloaded jemalloc w/ Unsafe Patch by Robert Stupp; Reviewed by Ariel Weisberg for CASSANDRA-8714 Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/0d2ec11c Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/0d2ec11c Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/0d2ec11c Branch: refs/heads/trunk Commit: 0d2ec11c7e0abfb84d872289af6d3ac386cf381f Parents: 8fe1fdf Author: Robert Stupp <[email protected]> Authored: Thu Mar 5 10:53:56 2015 +0100 Committer: Robert Stupp <[email protected]> Committed: Thu Mar 5 10:57:03 2015 +0100 ---------------------------------------------------------------------- CHANGES.txt | 1 + NEWS.txt | 3 + build.xml | 2 +- conf/cassandra-env.sh | 62 ++++++ lib/licenses/ohc-0.3.2.txt | 201 ------------------- lib/licenses/ohc-0.3.4.txt | 201 +++++++++++++++++++ lib/ohc-core-0.3.2.jar | Bin 148339 -> 0 bytes lib/ohc-core-0.3.4.jar | Bin 0 -> 145954 bytes lib/ohc-core-j8-0.3.2.jar | Bin 5079 -> 0 bytes lib/ohc-core-j8-0.3.4.jar | Bin 0 -> 5081 bytes .../org/apache/cassandra/config/Config.java | 5 +- .../cassandra/config/DatabaseDescriptor.java | 10 - .../apache/cassandra/io/util/IAllocator.java | 24 --- .../cassandra/io/util/JEMallocAllocator.java | 48 ----- .../org/apache/cassandra/io/util/Memory.java | 28 ++- .../cassandra/io/util/NativeAllocator.java | 51 ----- .../apache/cassandra/io/util/SafeMemory.java | 3 +- .../org/apache/cassandra/utils/FBUtilities.java | 10 +- .../cassandra/utils/memory/MemoryUtil.java | 14 +- .../cassandra/utils/memory/NativeAllocator.java | 12 +- 20 files changed, 311 insertions(+), 364 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cassandra/blob/0d2ec11c/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index 3d6cc46..c4029e7 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,4 +1,5 @@ 3.0 + * use preloaded jemalloc w/ Unsafe (CASSANDRA-8714) * Add role based access control (CASSANDRA-7653, 8650, 7216, 8760, 8849) * Avoid accessing partitioner through StorageProxy (CASSANDRA-8244, 8268) * Upgrade Metrics library and remove depricated metrics (CASSANDRA-5657) http://git-wip-us.apache.org/repos/asf/cassandra/blob/0d2ec11c/NEWS.txt ---------------------------------------------------------------------- diff --git a/NEWS.txt b/NEWS.txt index 73da5ba..458618e 100644 --- a/NEWS.txt +++ b/NEWS.txt @@ -39,6 +39,9 @@ New features directory. - Support for user-defined functions and user-defined aggregates have been added to CQL. + - Row-cache is now fully off-heap. + - jemalloc is now automatically preloaded and used on Linux and OS-X if + installed. Upgrading http://git-wip-us.apache.org/repos/asf/cassandra/blob/0d2ec11c/build.xml ---------------------------------------------------------------------- diff --git a/build.xml b/build.xml index 6305556..be067af 100644 --- a/build.xml +++ b/build.xml @@ -378,7 +378,7 @@ <dependency groupId="com.clearspring.analytics" artifactId="stream" version="2.5.2" /> <dependency groupId="com.datastax.cassandra" artifactId="cassandra-driver-core" version="2.1.2" /> <dependency groupId="org.javassist" artifactId="javassist" version="3.18.2-GA" /> - <dependency groupId="org.caffinitas.ohc" artifactId="ohc-core" version="0.3.2" /> + <dependency groupId="org.caffinitas.ohc" artifactId="ohc-core" version="0.3.4" /> <dependency groupId="net.sf.supercsv" artifactId="super-csv" version="2.1.0" /> <dependency groupId="net.ju-n.compile-command-annotations" artifactId="compile-command-annotations" version="1.2.0" /> <dependency groupId="org.fusesource" artifactId="sigar" version="1.6.4"> http://git-wip-us.apache.org/repos/asf/cassandra/blob/0d2ec11c/conf/cassandra-env.sh ---------------------------------------------------------------------- diff --git a/conf/cassandra-env.sh b/conf/cassandra-env.sh index 2fb3a1c..4eb4c88 100644 --- a/conf/cassandra-env.sh +++ b/conf/cassandra-env.sh @@ -160,6 +160,68 @@ then export MALLOC_ARENA_MAX=4 fi +# Cassandra uses an installed jemalloc via LD_PRELOAD / DYLD_INSERT_LIBRARIES by default to improve off-heap +# memory allocation performance. The following code searches for an installed libjemalloc.dylib/.so/.1.so using +# Linux and OS-X specific approaches. +# To specify your own libjemalloc in a different path, configure the fully qualified path in CASSANDRA_LIBJEMALLOC. +# To disable jemalloc at all set CASSANDRA_LIBJEMALLOC=- +# +#CASSANDRA_LIBJEMALLOC= +# +find_library() +{ + lname=$1 + shift + lext=$1 + shift + while [ ! -z $1 ] ; do + path=$1 + shift + for dir in $(echo $path | tr ":" " ") ; do + if [ -d $dir ] ; then + if [ -f $dir/$lname$lext ] ; then + echo $dir/$lname$lext + return + fi + fi + done + done +} +case "`uname -s`" in + Linux) + if [ -z $CASSANDRA_LIBJEMALLOC ] ; then + which ldconfig > /dev/null 2>&1 + if [ $? = 0 ] ; then + # e.g. for CentOS + dirs=`ldconfig -v 2>/dev/null | grep -v ^$'\t' | sed 's/^\([^:]*\):.*$/\1/'` + else + # e.g. for Debian, OpenSUSE + dirs="/lib64 /lib /usr/lib64 /usr/lib `cat /etc/ld.so.conf /etc/ld.so.conf.d/*.conf | grep '^/'`" + fi + CASSANDRA_LIBJEMALLOC=$(find_library libjemalloc .so $dirs) + fi + if [ -z $CASSANDRA_LIBJEMALLOC ] ; then + CASSANDRA_LIBJEMALLOC=$(find_library libjemalloc .so.1 $dirs) + fi + if [ ! -z $CASSANDRA_LIBJEMALLOC ] ; then + if [ "-" != "$CASSANDRA_LIBJEMALLOC" ] ; then + export LD_PRELOAD=$CASSANDRA_LIBJEMALLOC + fi + fi + ;; + Darwin) + if [ -z $CASSANDRA_LIBJEMALLOC ] ; then + CASSANDRA_LIBJEMALLOC=$(find_library libjemalloc .dylib $DYLD_LIBRARY_PATH ${DYLD_FALLBACK_LIBRARY_PATH-$HOME/lib:/usr/local/lib:/lib:/usr/lib}) + fi + if [ ! -z $CASSANDRA_LIBJEMALLOC ] ; then + if [ "-" != "$CASSANDRA_LIBJEMALLOC" ] ; then + export DYLD_INSERT_LIBRARIES=$CASSANDRA_LIBJEMALLOC + fi + fi + ;; +esac + + # Specifies the default port over which Cassandra will be available for # JMX connections. # For security reasons, you should not expose this port to the internet. Firewall it if needed. http://git-wip-us.apache.org/repos/asf/cassandra/blob/0d2ec11c/lib/licenses/ohc-0.3.2.txt ---------------------------------------------------------------------- diff --git a/lib/licenses/ohc-0.3.2.txt b/lib/licenses/ohc-0.3.2.txt deleted file mode 100644 index eb6b5d3..0000000 --- a/lib/licenses/ohc-0.3.2.txt +++ /dev/null @@ -1,201 +0,0 @@ - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright 2014 Robert Stupp, Koeln, Germany, robert-stupp.de - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. http://git-wip-us.apache.org/repos/asf/cassandra/blob/0d2ec11c/lib/licenses/ohc-0.3.4.txt ---------------------------------------------------------------------- diff --git a/lib/licenses/ohc-0.3.4.txt b/lib/licenses/ohc-0.3.4.txt new file mode 100644 index 0000000..eb6b5d3 --- /dev/null +++ b/lib/licenses/ohc-0.3.4.txt @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2014 Robert Stupp, Koeln, Germany, robert-stupp.de + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. http://git-wip-us.apache.org/repos/asf/cassandra/blob/0d2ec11c/lib/ohc-core-0.3.2.jar ---------------------------------------------------------------------- diff --git a/lib/ohc-core-0.3.2.jar b/lib/ohc-core-0.3.2.jar deleted file mode 100644 index c9ef6a1..0000000 Binary files a/lib/ohc-core-0.3.2.jar and /dev/null differ http://git-wip-us.apache.org/repos/asf/cassandra/blob/0d2ec11c/lib/ohc-core-0.3.4.jar ---------------------------------------------------------------------- diff --git a/lib/ohc-core-0.3.4.jar b/lib/ohc-core-0.3.4.jar new file mode 100644 index 0000000..0773e78 Binary files /dev/null and b/lib/ohc-core-0.3.4.jar differ http://git-wip-us.apache.org/repos/asf/cassandra/blob/0d2ec11c/lib/ohc-core-j8-0.3.2.jar ---------------------------------------------------------------------- diff --git a/lib/ohc-core-j8-0.3.2.jar b/lib/ohc-core-j8-0.3.2.jar deleted file mode 100644 index b14ffc5..0000000 Binary files a/lib/ohc-core-j8-0.3.2.jar and /dev/null differ http://git-wip-us.apache.org/repos/asf/cassandra/blob/0d2ec11c/lib/ohc-core-j8-0.3.4.jar ---------------------------------------------------------------------- diff --git a/lib/ohc-core-j8-0.3.4.jar b/lib/ohc-core-j8-0.3.4.jar new file mode 100644 index 0000000..faa102f Binary files /dev/null and b/lib/ohc-core-j8-0.3.4.jar differ http://git-wip-us.apache.org/repos/asf/cassandra/blob/0d2ec11c/src/java/org/apache/cassandra/config/Config.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/config/Config.java b/src/java/org/apache/cassandra/config/Config.java index eec6826..163c4d6 100644 --- a/src/java/org/apache/cassandra/config/Config.java +++ b/src/java/org/apache/cassandra/config/Config.java @@ -28,8 +28,6 @@ import com.google.common.collect.Sets; import org.apache.cassandra.config.EncryptionOptions.ClientEncryptionOptions; import org.apache.cassandra.config.EncryptionOptions.ServerEncryptionOptions; import org.apache.cassandra.exceptions.ConfigurationException; -import org.apache.cassandra.io.util.NativeAllocator; -import org.apache.cassandra.utils.FBUtilities; import org.supercsv.io.CsvListReader; import org.supercsv.prefs.CsvPreference; @@ -205,7 +203,8 @@ public class Config public volatile int counter_cache_save_period = 7200; public volatile int counter_cache_keys_to_save = Integer.MAX_VALUE; - public String memory_allocator = NativeAllocator.class.getSimpleName(); + @Deprecated + public String memory_allocator; public Integer file_cache_size_in_mb; http://git-wip-us.apache.org/repos/asf/cassandra/blob/0d2ec11c/src/java/org/apache/cassandra/config/DatabaseDescriptor.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java index 92d5d5b..b7acbb9 100644 --- a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java +++ b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java @@ -39,7 +39,6 @@ import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.io.FSWriteError; import org.apache.cassandra.io.sstable.format.SSTableFormat; import org.apache.cassandra.io.util.FileUtils; -import org.apache.cassandra.io.util.IAllocator; import org.apache.cassandra.locator.*; import org.apache.cassandra.net.MessagingService; import org.apache.cassandra.scheduler.IRequestScheduler; @@ -48,7 +47,6 @@ import org.apache.cassandra.service.CacheService; import org.apache.cassandra.thrift.ThriftServer; import org.apache.cassandra.utils.ByteBufferUtil; import org.apache.cassandra.utils.FBUtilities; -import org.apache.cassandra.utils.JVMStabilityInspector; import org.apache.cassandra.utils.memory.*; public class DatabaseDescriptor @@ -89,7 +87,6 @@ public class DatabaseDescriptor private static long keyCacheSizeInMB; private static long counterCacheSizeInMB; - private static IAllocator memoryAllocator; private static long indexSummaryCapacityInMB; private static String localDC; @@ -562,8 +559,6 @@ public class DatabaseDescriptor throw new ConfigurationException("index_summary_capacity_in_mb option was set incorrectly to '" + conf.index_summary_capacity_in_mb + "', it should be a non-negative integer.", false); - memoryAllocator = FBUtilities.newOffHeapAllocator(conf.memory_allocator); - if(conf.encryption_options != null) { logger.warn("Please rename encryption_options as server_encryption_options in the yaml"); @@ -1491,11 +1486,6 @@ public class DatabaseDescriptor conf.counter_cache_keys_to_save = counterCacheKeysToSave; } - public static IAllocator getoffHeapMemoryAllocator() - { - return memoryAllocator; - } - public static int getStreamingSocketTimeout() { return conf.streaming_socket_timeout_in_ms; http://git-wip-us.apache.org/repos/asf/cassandra/blob/0d2ec11c/src/java/org/apache/cassandra/io/util/IAllocator.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/io/util/IAllocator.java b/src/java/org/apache/cassandra/io/util/IAllocator.java deleted file mode 100644 index d633617..0000000 --- a/src/java/org/apache/cassandra/io/util/IAllocator.java +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.cassandra.io.util; - -public interface IAllocator -{ - long allocate(long size); - void free(long peer); -} http://git-wip-us.apache.org/repos/asf/cassandra/blob/0d2ec11c/src/java/org/apache/cassandra/io/util/JEMallocAllocator.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/io/util/JEMallocAllocator.java b/src/java/org/apache/cassandra/io/util/JEMallocAllocator.java deleted file mode 100644 index 07d19fe..0000000 --- a/src/java/org/apache/cassandra/io/util/JEMallocAllocator.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.cassandra.io.util; - -import com.sun.jna.Library; -import com.sun.jna.Native; - -public class JEMallocAllocator implements IAllocator -{ - public interface JEMLibrary extends Library - { - long malloc(long size); - - void free(long pointer); - } - - private final JEMLibrary library; - - public JEMallocAllocator() - { - library = (JEMLibrary) Native.loadLibrary("jemalloc", JEMLibrary.class); - } - - public long allocate(long size) - { - return library.malloc(size); - } - - public void free(long peer) - { - library.free(peer); - } -} http://git-wip-us.apache.org/repos/asf/cassandra/blob/0d2ec11c/src/java/org/apache/cassandra/io/util/Memory.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/io/util/Memory.java b/src/java/org/apache/cassandra/io/util/Memory.java index add5dee..e12064d 100644 --- a/src/java/org/apache/cassandra/io/util/Memory.java +++ b/src/java/org/apache/cassandra/io/util/Memory.java @@ -17,12 +17,11 @@ */ package org.apache.cassandra.io.util; +import java.lang.reflect.Field; import java.nio.ByteBuffer; import java.nio.ByteOrder; -import com.sun.jna.Native; import net.nicoulaj.compilecommand.annotations.Inline; -import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.utils.FastByteOperations; import org.apache.cassandra.utils.concurrent.Ref; import org.apache.cassandra.utils.memory.MemoryUtil; @@ -34,13 +33,28 @@ import sun.nio.ch.DirectBuffer; */ public class Memory implements AutoCloseable { - private static final Unsafe unsafe = NativeAllocator.unsafe; - static final IAllocator allocator = DatabaseDescriptor.getoffHeapMemoryAllocator(); + private static final Unsafe unsafe; + static + { + try + { + Field field = sun.misc.Unsafe.class.getDeclaredField("theUnsafe"); + field.setAccessible(true); + unsafe = (sun.misc.Unsafe) field.get(null); + } + catch (Exception e) + { + throw new AssertionError(e); + } + } + private static final long BYTE_ARRAY_BASE_OFFSET = unsafe.arrayBaseOffset(byte[].class); private static final boolean bigEndian = ByteOrder.nativeOrder().equals(ByteOrder.BIG_ENDIAN); private static final boolean unaligned; + public static final ByteBuffer[] NO_BYTE_BUFFERS = new ByteBuffer[0]; + static { String arch = System.getProperty("os.arch"); @@ -57,7 +71,7 @@ public class Memory implements AutoCloseable if (bytes <= 0) throw new AssertionError(); size = bytes; - peer = allocator.allocate(size); + peer = MemoryUtil.allocate(size); // we permit a 0 peer iff size is zero, since such an allocation makes no sense, and an allocator would be // justified in returning a null pointer (and permitted to do so: http://www.cplusplus.com/reference/cstdlib/malloc) if (peer == 0) @@ -341,7 +355,7 @@ public class Memory implements AutoCloseable public void free() { - if (peer != 0) allocator.free(peer); + if (peer != 0) MemoryUtil.free(peer); else assert size == 0; peer = 0; } @@ -373,7 +387,7 @@ public class Memory implements AutoCloseable public ByteBuffer[] asByteBuffers(long offset, long length) { if (size() == 0) - return new ByteBuffer[0]; + return NO_BYTE_BUFFERS; ByteBuffer[] result = new ByteBuffer[(int) (length / Integer.MAX_VALUE) + 1]; int size = (int) (size() / result.length); http://git-wip-us.apache.org/repos/asf/cassandra/blob/0d2ec11c/src/java/org/apache/cassandra/io/util/NativeAllocator.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/io/util/NativeAllocator.java b/src/java/org/apache/cassandra/io/util/NativeAllocator.java deleted file mode 100644 index da0b362..0000000 --- a/src/java/org/apache/cassandra/io/util/NativeAllocator.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.cassandra.io.util; - -import java.lang.reflect.Field; - -import sun.misc.Unsafe; - -public class NativeAllocator implements IAllocator -{ - static final Unsafe unsafe; - static - { - try - { - Field field = sun.misc.Unsafe.class.getDeclaredField("theUnsafe"); - field.setAccessible(true); - unsafe = (sun.misc.Unsafe) field.get(null); - } - catch (Exception e) - { - throw new AssertionError(e); - } - } - - public long allocate(long size) - { - return unsafe.allocateMemory(size); - } - - public void free(long peer) - { - unsafe.freeMemory(peer); - } - -} http://git-wip-us.apache.org/repos/asf/cassandra/blob/0d2ec11c/src/java/org/apache/cassandra/io/util/SafeMemory.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/io/util/SafeMemory.java b/src/java/org/apache/cassandra/io/util/SafeMemory.java index 74a1f1e..c933e40 100644 --- a/src/java/org/apache/cassandra/io/util/SafeMemory.java +++ b/src/java/org/apache/cassandra/io/util/SafeMemory.java @@ -22,6 +22,7 @@ import net.nicoulaj.compilecommand.annotations.Inline; import org.apache.cassandra.utils.concurrent.Ref; import org.apache.cassandra.utils.concurrent.RefCounted; import org.apache.cassandra.utils.concurrent.SharedCloseable; +import org.apache.cassandra.utils.memory.MemoryUtil; public class SafeMemory extends Memory implements SharedCloseable { @@ -80,7 +81,7 @@ public class SafeMemory extends Memory implements SharedCloseable { /** see {@link Memory#Memory(long)} re: null pointers*/ if (peer != 0) - Memory.allocator.free(peer); + MemoryUtil.free(peer); } public String name() http://git-wip-us.apache.org/repos/asf/cassandra/blob/0d2ec11c/src/java/org/apache/cassandra/utils/FBUtilities.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/utils/FBUtilities.java b/src/java/org/apache/cassandra/utils/FBUtilities.java index 64e4987..f8a0ea7 100644 --- a/src/java/org/apache/cassandra/utils/FBUtilities.java +++ b/src/java/org/apache/cassandra/utils/FBUtilities.java @@ -50,7 +50,6 @@ import org.apache.cassandra.io.IVersionedSerializer; import org.apache.cassandra.io.compress.CompressionParameters; import org.apache.cassandra.io.util.DataOutputBuffer; import org.apache.cassandra.io.util.FileUtils; -import org.apache.cassandra.io.util.IAllocator; import org.apache.cassandra.net.AsyncOneResponse; import org.apache.thrift.*; import org.codehaus.jackson.JsonFactory; @@ -60,7 +59,7 @@ public class FBUtilities { private static final Logger logger = LoggerFactory.getLogger(FBUtilities.class); - private static ObjectMapper jsonMapper = new ObjectMapper(new JsonFactory()); + private static final ObjectMapper jsonMapper = new ObjectMapper(new JsonFactory()); public static final BigInteger TWO = new BigInteger("2"); private static final String DEFAULT_TRIGGER_DIR = "triggers"; @@ -413,13 +412,6 @@ public class FBUtilities return FBUtilities.instanceOrConstruct(partitionerClassName, "partitioner"); } - public static IAllocator newOffHeapAllocator(String offheap_allocator) throws ConfigurationException - { - if (!offheap_allocator.contains(".")) - offheap_allocator = "org.apache.cassandra.io.util." + offheap_allocator; - return FBUtilities.construct(offheap_allocator, "off-heap allocator"); - } - public static IAuthorizer newAuthorizer(String className) throws ConfigurationException { if (!className.contains(".")) http://git-wip-us.apache.org/repos/asf/cassandra/blob/0d2ec11c/src/java/org/apache/cassandra/utils/memory/MemoryUtil.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/utils/memory/MemoryUtil.java b/src/java/org/apache/cassandra/utils/memory/MemoryUtil.java index 159618e..60b2b7f 100644 --- a/src/java/org/apache/cassandra/utils/memory/MemoryUtil.java +++ b/src/java/org/apache/cassandra/utils/memory/MemoryUtil.java @@ -22,7 +22,9 @@ import java.nio.Buffer; import java.nio.ByteBuffer; import java.nio.ByteOrder; +import com.sun.jna.Native; import sun.misc.Unsafe; +import sun.nio.ch.DirectBuffer; public abstract class MemoryUtil { @@ -64,6 +66,16 @@ public abstract class MemoryUtil } } + public static long allocate(long size) + { + return Native.malloc(size); + } + + public static void free(long peer) + { + Native.free(peer); + } + public static void setByte(long address, byte b) { unsafe.putByte(address, b); @@ -240,7 +252,7 @@ public abstract class MemoryUtil return; if (buffer.isDirect()) - setBytes(unsafe.getLong(buffer, DIRECT_BYTE_BUFFER_ADDRESS_OFFSET) + start, address, count); + setBytes(((DirectBuffer)buffer).address() + start, address, count); else setBytes(address, buffer.array(), buffer.arrayOffset() + start, count); } http://git-wip-us.apache.org/repos/asf/cassandra/blob/0d2ec11c/src/java/org/apache/cassandra/utils/memory/NativeAllocator.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/utils/memory/NativeAllocator.java b/src/java/org/apache/cassandra/utils/memory/NativeAllocator.java index aca4133..be78c5d 100644 --- a/src/java/org/apache/cassandra/utils/memory/NativeAllocator.java +++ b/src/java/org/apache/cassandra/utils/memory/NativeAllocator.java @@ -25,7 +25,6 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; import org.apache.cassandra.config.CFMetaData; -import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.db.Cell; import org.apache.cassandra.db.CounterCell; import org.apache.cassandra.db.DecoratedKey; @@ -36,7 +35,6 @@ import org.apache.cassandra.db.NativeCounterCell; import org.apache.cassandra.db.NativeDecoratedKey; import org.apache.cassandra.db.NativeDeletedCell; import org.apache.cassandra.db.NativeExpiringCell; -import org.apache.cassandra.io.util.IAllocator; import org.apache.cassandra.utils.concurrent.OpOrder; public class NativeAllocator extends MemtableAllocator @@ -45,8 +43,6 @@ public class NativeAllocator extends MemtableAllocator private final static int MAX_CLONED_SIZE = 128 * 1024; // bigger than this don't go in the region private final static int MIN_REGION_SIZE = 8 * 1024; - private static final IAllocator allocator = DatabaseDescriptor.getoffHeapMemoryAllocator(); - // globally stash any Regions we allocate but are beaten to using, and use these up before allocating any more private static final Map<Integer, RaceAllocated> RACE_ALLOCATED = new HashMap<>(); @@ -137,21 +133,21 @@ public class NativeAllocator extends MemtableAllocator // if there are none, we allocate one if (next == null) - next = new Region(allocator.allocate(size), size); + next = new Region(MemoryUtil.allocate(size), size); // we try to swap in the region we've obtained; // if we fail to swap the region, we try to stash it for repurposing later; if we're out of stash room, we free it if (currentRegion.compareAndSet(current, next)) regions.add(next); else if (!raceAllocated.stash(next)) - allocator.free(next.peer); + MemoryUtil.free(next.peer); } private long allocateOversize(int size, OpOrder.Group opGroup) { // satisfy large allocations directly from JVM since they don't cause fragmentation // as badly, and fill up our regions quickly - Region region = new Region(allocator.allocate(size), size); + Region region = new Region(MemoryUtil.allocate(size), size); regions.add(region); long peer; @@ -164,7 +160,7 @@ public class NativeAllocator extends MemtableAllocator public void setDiscarded() { for (Region region : regions) - allocator.free(region.peer); + MemoryUtil.free(region.peer); super.setDiscarded(); }
