http://git-wip-us.apache.org/repos/asf/cassandra/blob/72790dc8/lib/licenses/jflex-1.6.0.txt ---------------------------------------------------------------------- diff --git a/lib/licenses/jflex-1.6.0.txt b/lib/licenses/jflex-1.6.0.txt new file mode 100644 index 0000000..50086f8 --- /dev/null +++ b/lib/licenses/jflex-1.6.0.txt @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. \ No newline at end of file
http://git-wip-us.apache.org/repos/asf/cassandra/blob/72790dc8/lib/licenses/primitive-1.0.txt ---------------------------------------------------------------------- diff --git a/lib/licenses/primitive-1.0.txt b/lib/licenses/primitive-1.0.txt new file mode 100644 index 0000000..50086f8 --- /dev/null +++ b/lib/licenses/primitive-1.0.txt @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. \ No newline at end of file http://git-wip-us.apache.org/repos/asf/cassandra/blob/72790dc8/lib/licenses/snowball-stemmer-1.3.0.581.1.txt ---------------------------------------------------------------------- diff --git a/lib/licenses/snowball-stemmer-1.3.0.581.1.txt b/lib/licenses/snowball-stemmer-1.3.0.581.1.txt new file mode 100644 index 0000000..50086f8 --- /dev/null +++ b/lib/licenses/snowball-stemmer-1.3.0.581.1.txt @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. \ No newline at end of file http://git-wip-us.apache.org/repos/asf/cassandra/blob/72790dc8/lib/primitive-1.0.jar ---------------------------------------------------------------------- diff --git a/lib/primitive-1.0.jar b/lib/primitive-1.0.jar new file mode 100644 index 0000000..288daa0 Binary files /dev/null and b/lib/primitive-1.0.jar differ http://git-wip-us.apache.org/repos/asf/cassandra/blob/72790dc8/lib/snowball-stemmer-1.3.0.581.1.jar ---------------------------------------------------------------------- diff --git a/lib/snowball-stemmer-1.3.0.581.1.jar b/lib/snowball-stemmer-1.3.0.581.1.jar new file mode 100644 index 0000000..92189b9 Binary files /dev/null and b/lib/snowball-stemmer-1.3.0.581.1.jar differ http://git-wip-us.apache.org/repos/asf/cassandra/blob/72790dc8/src/java/org/apache/cassandra/config/DatabaseDescriptor.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java index 116d92e..2a2719a 100644 --- a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java +++ b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java @@ -1964,5 +1964,10 @@ public class DatabaseDescriptor public static void setEncryptionContext(EncryptionContext ec) { encryptionContext = ec; - } + } + + public static int searchConcurrencyFactor() + { + return Integer.valueOf(System.getProperty("cassandra.search_concurrency_factor", "1")); + } } http://git-wip-us.apache.org/repos/asf/cassandra/blob/72790dc8/src/java/org/apache/cassandra/db/ColumnIndex.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/db/ColumnIndex.java b/src/java/org/apache/cassandra/db/ColumnIndex.java index 749c155..930fc05 100644 --- a/src/java/org/apache/cassandra/db/ColumnIndex.java +++ b/src/java/org/apache/cassandra/db/ColumnIndex.java @@ -152,9 +152,9 @@ public class ColumnIndex UnfilteredSerializer.serializer.serialize(unfiltered, header, writer, pos - previousRowStart, version); - // notify observers about each new cell added to the row - if (!observers.isEmpty() && unfiltered.isRow()) - ((Row) unfiltered).stream().forEach(cell -> observers.forEach((o) -> o.nextCell(cell))); + // notify observers about each new row + if (!observers.isEmpty()) + observers.forEach((o) -> o.nextUnfilteredCluster(unfiltered)); lastClustering = unfiltered.clustering(); previousRowStart = pos; http://git-wip-us.apache.org/repos/asf/cassandra/blob/72790dc8/src/java/org/apache/cassandra/db/filter/RowFilter.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/db/filter/RowFilter.java b/src/java/org/apache/cassandra/db/filter/RowFilter.java index 4cd2f64..c234fc9 100644 --- a/src/java/org/apache/cassandra/db/filter/RowFilter.java +++ b/src/java/org/apache/cassandra/db/filter/RowFilter.java @@ -171,6 +171,11 @@ public abstract class RowFilter implements Iterable<RowFilter.Expression> return withNewExpressions(newExpressions); } + public RowFilter withoutExpressions() + { + return withNewExpressions(Collections.emptyList()); + } + protected abstract RowFilter withNewExpressions(List<Expression> expressions); public boolean isEmpty() @@ -312,7 +317,7 @@ public abstract class RowFilter implements Iterable<RowFilter.Expression> // Note: the order of this enum matter, it's used for serialization protected enum Kind { SIMPLE, MAP_EQUALITY, THRIFT_DYN_EXPR, CUSTOM } - abstract Kind kind(); + protected abstract Kind kind(); protected final ColumnDefinition column; protected final Operator operator; protected final ByteBuffer value; @@ -689,7 +694,7 @@ public abstract class RowFilter implements Iterable<RowFilter.Expression> } @Override - Kind kind() + protected Kind kind() { return Kind.SIMPLE; } @@ -782,7 +787,7 @@ public abstract class RowFilter implements Iterable<RowFilter.Expression> } @Override - Kind kind() + protected Kind kind() { return Kind.MAP_EQUALITY; } @@ -833,7 +838,7 @@ public abstract class RowFilter implements Iterable<RowFilter.Expression> } @Override - Kind kind() + protected Kind kind() { return Kind.THRIFT_DYN_EXPR; } @@ -883,7 +888,7 @@ public abstract class RowFilter implements Iterable<RowFilter.Expression> .customExpressionValueType()); } - Kind kind() + protected Kind kind() { return Kind.CUSTOM; } http://git-wip-us.apache.org/repos/asf/cassandra/blob/72790dc8/src/java/org/apache/cassandra/index/SecondaryIndexManager.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/index/SecondaryIndexManager.java b/src/java/org/apache/cassandra/index/SecondaryIndexManager.java index 7349167..e4a03de 100644 --- a/src/java/org/apache/cassandra/index/SecondaryIndexManager.java +++ b/src/java/org/apache/cassandra/index/SecondaryIndexManager.java @@ -665,6 +665,11 @@ public class SecondaryIndexManager implements IndexRegistry return selected; } + public Optional<Index> getBestIndexFor(RowFilter.Expression expression) + { + return indexes.values().stream().filter((i) -> i.supportsExpression(expression.column(), expression.operator())).findFirst(); + } + /** * Called at write time to ensure that values present in the update * are valid according to the rules of all registered indexes which @@ -1040,6 +1045,12 @@ public class SecondaryIndexManager implements IndexRegistry private static void executeAllBlocking(Stream<Index> indexers, Function<Index, Callable<?>> function) { + if (function == null) + { + logger.error("failed to flush indexes: {} because flush task is missing.", indexers); + return; + } + List<Future<?>> waitFor = new ArrayList<>(); indexers.forEach(indexer -> { Callable<?> task = function.apply(indexer); http://git-wip-us.apache.org/repos/asf/cassandra/blob/72790dc8/src/java/org/apache/cassandra/index/sasi/SASIIndex.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/index/sasi/SASIIndex.java b/src/java/org/apache/cassandra/index/sasi/SASIIndex.java new file mode 100644 index 0000000..d69b440 --- /dev/null +++ b/src/java/org/apache/cassandra/index/sasi/SASIIndex.java @@ -0,0 +1,288 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.index.sasi; + +import java.util.*; +import java.util.concurrent.Callable; +import java.util.function.BiFunction; + +import com.googlecode.concurrenttrees.common.Iterables; +import org.apache.cassandra.config.CFMetaData; +import org.apache.cassandra.config.ColumnDefinition; +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.config.Schema; +import org.apache.cassandra.cql3.Operator; +import org.apache.cassandra.db.*; +import org.apache.cassandra.db.compaction.OperationType; +import org.apache.cassandra.db.filter.RowFilter; +import org.apache.cassandra.db.marshal.AbstractType; +import org.apache.cassandra.db.partitions.PartitionIterator; +import org.apache.cassandra.db.partitions.PartitionUpdate; +import org.apache.cassandra.db.rows.Row; +import org.apache.cassandra.exceptions.InvalidRequestException; +import org.apache.cassandra.index.Index; +import org.apache.cassandra.index.IndexRegistry; +import org.apache.cassandra.index.SecondaryIndexBuilder; +import org.apache.cassandra.index.internal.CassandraIndex; +import org.apache.cassandra.index.sasi.conf.ColumnIndex; +import org.apache.cassandra.index.sasi.disk.PerSSTableIndexWriter; +import org.apache.cassandra.index.sasi.plan.QueryPlan; +import org.apache.cassandra.index.transactions.IndexTransaction; +import org.apache.cassandra.io.sstable.Descriptor; +import org.apache.cassandra.io.sstable.format.SSTableFlushObserver; +import org.apache.cassandra.io.sstable.format.SSTableReader; +import org.apache.cassandra.notifications.*; +import org.apache.cassandra.schema.IndexMetadata; +import org.apache.cassandra.utils.FBUtilities; +import org.apache.cassandra.utils.concurrent.OpOrder; + +public class SASIIndex implements Index, INotificationConsumer +{ + private static class SASIIndexBuildingSupport implements IndexBuildingSupport + { + public SecondaryIndexBuilder getIndexBuildTask(ColumnFamilyStore cfs, + Set<Index> indexes, + Collection<SSTableReader> sstablesToRebuild) + { + NavigableMap<SSTableReader, Map<ColumnDefinition, ColumnIndex>> sstables = new TreeMap<>((a, b) -> { + return Integer.compare(a.descriptor.generation, b.descriptor.generation); + }); + + indexes.stream() + .filter((i) -> i instanceof SASIIndex) + .forEach((i) -> { + SASIIndex sasi = (SASIIndex) i; + sstablesToRebuild.stream() + .filter((sstable) -> !sasi.index.hasSSTable(sstable)) + .forEach((sstable) -> { + Map<ColumnDefinition, ColumnIndex> toBuild = sstables.get(sstable); + if (toBuild == null) + sstables.put(sstable, (toBuild = new HashMap<>())); + + toBuild.put(sasi.index.getDefinition(), sasi.index); + }); + }); + + return new SASIIndexBuilder(cfs, sstables); + } + } + + private static final SASIIndexBuildingSupport INDEX_BUILDER_SUPPORT = new SASIIndexBuildingSupport(); + + private final ColumnFamilyStore baseCfs; + private final IndexMetadata config; + private final ColumnIndex index; + + public SASIIndex(ColumnFamilyStore baseCfs, IndexMetadata config) + { + this.baseCfs = baseCfs; + this.config = config; + + ColumnDefinition column = CassandraIndex.parseTarget(baseCfs.metadata, config).left; + this.index = new ColumnIndex(baseCfs.metadata.getKeyValidator(), column, config); + + baseCfs.getTracker().subscribe(this); + } + + public static Map<String, String> validateOptions(Map<String, String> options) + { + return Collections.emptyMap(); + } + + public void register(IndexRegistry registry) + { + registry.registerIndex(this); + } + + public IndexMetadata getIndexMetadata() + { + return config; + } + + public Callable<?> getInitializationTask() + { + return null; + } + + public Callable<?> getMetadataReloadTask(IndexMetadata indexMetadata) + { + return null; + } + + public Callable<?> getBlockingFlushTask() + { + return null; // SASI indexes are flushed along side memtable + } + + public Callable<?> getInvalidateTask() + { + return getTruncateTask(FBUtilities.timestampMicros()); + } + + public Callable<?> getTruncateTask(long truncatedAt) + { + return () -> { + index.dropData(truncatedAt); + return null; + }; + } + + public boolean shouldBuildBlocking() + { + return true; + } + + public Optional<ColumnFamilyStore> getBackingTable() + { + return Optional.empty(); + } + + public boolean indexes(PartitionColumns columns) + { + return columns.contains(index.getDefinition()); + } + + public boolean dependsOn(ColumnDefinition column) + { + return index.getDefinition().compareTo(column) == 0; + } + + public boolean supportsExpression(ColumnDefinition column, Operator operator) + { + return dependsOn(column); + } + + public AbstractType<?> customExpressionValueType() + { + return null; + } + + public RowFilter getPostIndexQueryFilter(RowFilter filter) + { + return filter.withoutExpressions(); + } + + public long getEstimatedResultRows() + { + // this is temporary (until proper QueryPlan is integrated into Cassandra) + // and allows us to priority SASI indexes if any in the query since they + // are going to be more efficient, to query and intersect, than built-in indexes. + return Long.MIN_VALUE; + } + + public void validate(PartitionUpdate update) throws InvalidRequestException + {} + + public Indexer indexerFor(DecoratedKey key, PartitionColumns columns, int nowInSec, OpOrder.Group opGroup, IndexTransaction.Type transactionType) + { + return new Indexer() + { + public void begin() + {} + + public void partitionDelete(DeletionTime deletionTime) + {} + + public void rangeTombstone(RangeTombstone tombstone) + {} + + public void insertRow(Row row) + { + if (isNewData()) + adjustMemtableSize(index.index(key, row), opGroup); + } + + public void updateRow(Row oldRow, Row newRow) + { + insertRow(newRow); + } + + public void removeRow(Row row) + {} + + public void finish() + {} + + // we are only interested in the data from Memtable + // everything else is going to be handled by SSTableWriter observers + private boolean isNewData() + { + return transactionType == IndexTransaction.Type.UPDATE; + } + + public void adjustMemtableSize(long additionalSpace, OpOrder.Group opGroup) + { + baseCfs.getTracker().getView().getCurrentMemtable().getAllocator().onHeap().allocate(additionalSpace, opGroup); + } + }; + } + + public Searcher searcherFor(ReadCommand command) throws InvalidRequestException + { + CFMetaData config = command.metadata(); + ColumnFamilyStore cfs = Schema.instance.getColumnFamilyStoreInstance(config.cfId); + return controller -> new QueryPlan(cfs, command, DatabaseDescriptor.getRangeRpcTimeout()).execute(controller); + } + + public SSTableFlushObserver getFlushObserver(Descriptor descriptor, OperationType opType) + { + return newWriter(baseCfs.metadata.getKeyValidator(), descriptor, Collections.singletonMap(index.getDefinition(), index), opType); + } + + public BiFunction<PartitionIterator, ReadCommand, PartitionIterator> postProcessorFor(ReadCommand command) + { + return (partitionIterator, readCommand) -> partitionIterator; + } + + public IndexBuildingSupport getBuildTaskSupport() + { + return INDEX_BUILDER_SUPPORT; + } + + public void handleNotification(INotification notification, Object sender) + { + // unfortunately, we can only check the type of notification via instanceof :( + if (notification instanceof SSTableAddedNotification) + { + SSTableAddedNotification notice = (SSTableAddedNotification) notification; + index.update(Collections.<SSTableReader>emptyList(), Iterables.toList(notice.added)); + } + else if (notification instanceof SSTableListChangedNotification) + { + SSTableListChangedNotification notice = (SSTableListChangedNotification) notification; + index.update(notice.removed, notice.added); + } + else if (notification instanceof MemtableRenewedNotification) + { + index.switchMemtable(); + } + } + + public ColumnIndex getIndex() + { + return index; + } + + protected static PerSSTableIndexWriter newWriter(AbstractType<?> keyValidator, + Descriptor descriptor, + Map<ColumnDefinition, ColumnIndex> indexes, + OperationType opType) + { + return new PerSSTableIndexWriter(keyValidator, descriptor, opType, indexes); + } +} http://git-wip-us.apache.org/repos/asf/cassandra/blob/72790dc8/src/java/org/apache/cassandra/index/sasi/SASIIndexBuilder.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/index/sasi/SASIIndexBuilder.java b/src/java/org/apache/cassandra/index/sasi/SASIIndexBuilder.java new file mode 100644 index 0000000..fc5b675 --- /dev/null +++ b/src/java/org/apache/cassandra/index/sasi/SASIIndexBuilder.java @@ -0,0 +1,128 @@ +package org.apache.cassandra.index.sasi; + +import java.io.File; +import java.io.IOException; +import java.util.*; + +import org.apache.cassandra.config.ColumnDefinition; +import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.db.DecoratedKey; +import org.apache.cassandra.db.RowIndexEntry; +import org.apache.cassandra.db.compaction.CompactionInfo; +import org.apache.cassandra.db.compaction.CompactionInterruptedException; +import org.apache.cassandra.db.compaction.OperationType; +import org.apache.cassandra.db.marshal.AbstractType; +import org.apache.cassandra.index.SecondaryIndexBuilder; +import org.apache.cassandra.index.sasi.conf.ColumnIndex; +import org.apache.cassandra.index.sasi.disk.PerSSTableIndexWriter; +import org.apache.cassandra.io.FSReadError; +import org.apache.cassandra.io.sstable.KeyIterator; +import org.apache.cassandra.io.sstable.SSTable; +import org.apache.cassandra.io.sstable.SSTableIdentityIterator; +import org.apache.cassandra.io.sstable.format.SSTableReader; +import org.apache.cassandra.io.util.RandomAccessReader; +import org.apache.cassandra.utils.ByteBufferUtil; +import org.apache.cassandra.utils.UUIDGen; + +class SASIIndexBuilder extends SecondaryIndexBuilder +{ + private final ColumnFamilyStore cfs; + private final UUID compactionId = UUIDGen.getTimeUUID(); + + private final SortedMap<SSTableReader, Map<ColumnDefinition, ColumnIndex>> sstables; + + private long bytesProcessed = 0; + private final long totalSizeInBytes; + + public SASIIndexBuilder(ColumnFamilyStore cfs, SortedMap<SSTableReader, Map<ColumnDefinition, ColumnIndex>> sstables) + { + long totalIndexBytes = 0; + for (SSTableReader sstable : sstables.keySet()) + totalIndexBytes += getPrimaryIndexLength(sstable); + + this.cfs = cfs; + this.sstables = sstables; + this.totalSizeInBytes = totalIndexBytes; + } + + public void build() + { + AbstractType<?> keyValidator = cfs.metadata.getKeyValidator(); + for (Map.Entry<SSTableReader, Map<ColumnDefinition, ColumnIndex>> e : sstables.entrySet()) + { + SSTableReader sstable = e.getKey(); + Map<ColumnDefinition, ColumnIndex> indexes = e.getValue(); + + try (RandomAccessReader dataFile = sstable.openDataReader()) + { + PerSSTableIndexWriter indexWriter = SASIIndex.newWriter(keyValidator, sstable.descriptor, indexes, OperationType.COMPACTION); + + long previousKeyPosition = 0; + try (KeyIterator keys = new KeyIterator(sstable.descriptor, cfs.metadata)) + { + while (keys.hasNext()) + { + if (isStopRequested()) + throw new CompactionInterruptedException(getCompactionInfo()); + + final DecoratedKey key = keys.next(); + final long keyPosition = keys.getKeyPosition(); + + indexWriter.startPartition(key, keyPosition); + + try + { + RowIndexEntry indexEntry = sstable.getPosition(key, SSTableReader.Operator.EQ); + dataFile.seek(indexEntry.position + indexEntry.headerOffset()); + ByteBufferUtil.readWithShortLength(dataFile); // key + + try (SSTableIdentityIterator partition = new SSTableIdentityIterator(sstable, dataFile, key)) + { + while (partition.hasNext()) + indexWriter.nextUnfilteredCluster(partition.next()); + } + } + catch (IOException ex) + { + throw new FSReadError(ex, sstable.getFilename()); + } + + bytesProcessed += keyPosition - previousKeyPosition; + previousKeyPosition = keyPosition; + } + + completeSSTable(indexWriter, sstable, indexes.values()); + } + } + } + } + + public CompactionInfo getCompactionInfo() + { + return new CompactionInfo(cfs.metadata, + OperationType.INDEX_BUILD, + bytesProcessed, + totalSizeInBytes, + compactionId); + } + + private long getPrimaryIndexLength(SSTable sstable) + { + File primaryIndex = new File(sstable.getIndexFilename()); + return primaryIndex.exists() ? primaryIndex.length() : 0; + } + + private void completeSSTable(PerSSTableIndexWriter indexWriter, SSTableReader sstable, Collection<ColumnIndex> indexes) + { + indexWriter.complete(); + + for (ColumnIndex index : indexes) + { + File tmpIndex = new File(sstable.descriptor.filenameFor(index.getComponent())); + if (!tmpIndex.exists()) // no data was inserted into the index for given sstable + continue; + + index.update(Collections.<SSTableReader>emptyList(), Collections.singletonList(sstable)); + } + } +} http://git-wip-us.apache.org/repos/asf/cassandra/blob/72790dc8/src/java/org/apache/cassandra/index/sasi/SSTableIndex.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/index/sasi/SSTableIndex.java b/src/java/org/apache/cassandra/index/sasi/SSTableIndex.java new file mode 100644 index 0000000..7b65232 --- /dev/null +++ b/src/java/org/apache/cassandra/index/sasi/SSTableIndex.java @@ -0,0 +1,187 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.index.sasi; + +import java.io.File; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; + +import org.apache.cassandra.db.DecoratedKey; +import org.apache.cassandra.db.marshal.AbstractType; +import org.apache.cassandra.index.sasi.conf.ColumnIndex; +import org.apache.cassandra.index.sasi.disk.OnDiskIndex; +import org.apache.cassandra.index.sasi.disk.Token; +import org.apache.cassandra.index.sasi.plan.Expression; +import org.apache.cassandra.index.sasi.utils.RangeIterator; +import org.apache.cassandra.io.FSReadError; +import org.apache.cassandra.io.sstable.format.SSTableReader; +import org.apache.cassandra.io.util.FileUtils; +import org.apache.cassandra.utils.concurrent.Ref; + +import org.apache.commons.lang3.builder.HashCodeBuilder; + +import com.google.common.base.Function; + +public class SSTableIndex +{ + private final ColumnIndex columnIndex; + private final Ref<SSTableReader> sstableRef; + private final SSTableReader sstable; + private final OnDiskIndex index; + private final AtomicInteger references = new AtomicInteger(1); + private final AtomicBoolean obsolete = new AtomicBoolean(false); + + public SSTableIndex(ColumnIndex index, File indexFile, SSTableReader referent) + { + this.columnIndex = index; + this.sstableRef = referent.tryRef(); + this.sstable = sstableRef.get(); + + if (sstable == null) + throw new IllegalStateException("Couldn't acquire reference to the sstable: " + referent); + + AbstractType<?> validator = columnIndex.getValidator(); + + assert validator != null; + assert indexFile.exists() : String.format("SSTable %s should have index %s.", + sstable.getFilename(), + columnIndex.getIndexName()); + + this.index = new OnDiskIndex(indexFile, validator, new DecoratedKeyFetcher(sstable)); + } + + public ByteBuffer minTerm() + { + return index.minTerm(); + } + + public ByteBuffer maxTerm() + { + return index.maxTerm(); + } + + public ByteBuffer minKey() + { + return index.minKey(); + } + + public ByteBuffer maxKey() + { + return index.maxKey(); + } + + public RangeIterator<Long, Token> search(Expression expression) + { + return index.search(expression); + } + + public SSTableReader getSSTable() + { + return sstable; + } + + public String getPath() + { + return index.getIndexPath(); + } + + public boolean reference() + { + while (true) + { + int n = references.get(); + if (n <= 0) + return false; + if (references.compareAndSet(n, n + 1)) + return true; + } + } + + public void release() + { + int n = references.decrementAndGet(); + if (n == 0) + { + FileUtils.closeQuietly(index); + sstableRef.release(); + if (obsolete.get() || sstableRef.globalCount() == 0) + FileUtils.delete(index.getIndexPath()); + } + } + + public void markObsolete() + { + obsolete.getAndSet(true); + release(); + } + + public boolean isObsolete() + { + return obsolete.get(); + } + + public boolean equals(Object o) + { + return o instanceof SSTableIndex && index.getIndexPath().equals(((SSTableIndex) o).index.getIndexPath()); + } + + public int hashCode() + { + return new HashCodeBuilder().append(index.getIndexPath()).build(); + } + + public String toString() + { + return String.format("SSTableIndex(column: %s, SSTable: %s)", columnIndex.getColumnName(), sstable.descriptor); + } + + private static class DecoratedKeyFetcher implements Function<Long, DecoratedKey> + { + private final SSTableReader sstable; + + DecoratedKeyFetcher(SSTableReader reader) + { + sstable = reader; + } + + public DecoratedKey apply(Long offset) + { + try + { + return sstable.keyAt(offset); + } + catch (IOException e) + { + throw new FSReadError(new IOException("Failed to read key from " + sstable.descriptor, e), sstable.getFilename()); + } + } + + public int hashCode() + { + return sstable.descriptor.hashCode(); + } + + public boolean equals(Object other) + { + return other instanceof DecoratedKeyFetcher + && sstable.descriptor.equals(((DecoratedKeyFetcher) other).sstable.descriptor); + } + } +} http://git-wip-us.apache.org/repos/asf/cassandra/blob/72790dc8/src/java/org/apache/cassandra/index/sasi/Term.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/index/sasi/Term.java b/src/java/org/apache/cassandra/index/sasi/Term.java new file mode 100644 index 0000000..8e8ceb2 --- /dev/null +++ b/src/java/org/apache/cassandra/index/sasi/Term.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.index.sasi; + +import java.nio.ByteBuffer; + +import org.apache.cassandra.index.sasi.disk.OnDiskIndexBuilder.TermSize; +import org.apache.cassandra.index.sasi.utils.MappedBuffer; +import org.apache.cassandra.db.marshal.AbstractType; + +public class Term +{ + protected final MappedBuffer content; + protected final TermSize termSize; + + + public Term(MappedBuffer content, TermSize size) + { + this.content = content; + this.termSize = size; + } + + public ByteBuffer getTerm() + { + long offset = termSize.isConstant() ? content.position() : content.position() + 2; + int length = termSize.isConstant() ? termSize.size : content.getShort(content.position()); + + return content.getPageRegion(offset, length); + } + + public long getDataOffset() + { + long position = content.position(); + return position + (termSize.isConstant() ? termSize.size : 2 + content.getShort(position)); + } + + public int compareTo(AbstractType<?> comparator, ByteBuffer query) + { + return compareTo(comparator, query, true); + } + + public int compareTo(AbstractType<?> comparator, ByteBuffer query, boolean checkFully) + { + long position = content.position(); + int padding = termSize.isConstant() ? 0 : 2; + int len = termSize.isConstant() ? termSize.size : content.getShort(position); + + return content.comparePageTo(position + padding, checkFully ? len : Math.min(len, query.remaining()), comparator, query); + } +} http://git-wip-us.apache.org/repos/asf/cassandra/blob/72790dc8/src/java/org/apache/cassandra/index/sasi/TermIterator.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/index/sasi/TermIterator.java b/src/java/org/apache/cassandra/index/sasi/TermIterator.java new file mode 100644 index 0000000..cfa87c0 --- /dev/null +++ b/src/java/org/apache/cassandra/index/sasi/TermIterator.java @@ -0,0 +1,208 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.index.sasi; + +import java.util.List; +import java.util.Set; +import java.util.concurrent.*; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; + +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.index.sasi.disk.Token; +import org.apache.cassandra.index.sasi.plan.Expression; +import org.apache.cassandra.index.sasi.utils.RangeUnionIterator; +import org.apache.cassandra.index.sasi.utils.RangeIterator; +import org.apache.cassandra.io.util.FileUtils; + +import com.google.common.util.concurrent.MoreExecutors; +import com.google.common.util.concurrent.Uninterruptibles; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class TermIterator extends RangeIterator<Long, Token> +{ + private static final Logger logger = LoggerFactory.getLogger(TermIterator.class); + + private static final ThreadLocal<ExecutorService> SEARCH_EXECUTOR = new ThreadLocal<ExecutorService>() + { + public ExecutorService initialValue() + { + final String currentThread = Thread.currentThread().getName(); + final int concurrencyFactor = DatabaseDescriptor.searchConcurrencyFactor(); + + logger.info("Search Concurrency Factor is set to {} for {}", concurrencyFactor, currentThread); + + return (concurrencyFactor <= 1) + ? MoreExecutors.newDirectExecutorService() + : Executors.newFixedThreadPool(concurrencyFactor, new ThreadFactory() + { + public final AtomicInteger count = new AtomicInteger(); + + public Thread newThread(Runnable task) + { + return new Thread(task, currentThread + "-SEARCH-" + count.incrementAndGet()) {{ setDaemon(true); }}; + } + }); + } + }; + + private final Expression expression; + + private final RangeIterator<Long, Token> union; + private final Set<SSTableIndex> referencedIndexes; + + private TermIterator(Expression e, + RangeIterator<Long, Token> union, + Set<SSTableIndex> referencedIndexes) + { + super(union.getMinimum(), union.getMaximum(), union.getCount()); + + this.expression = e; + this.union = union; + this.referencedIndexes = referencedIndexes; + } + + public static TermIterator build(final Expression e, Set<SSTableIndex> perSSTableIndexes) + { + final List<RangeIterator<Long, Token>> tokens = new CopyOnWriteArrayList<>(); + final AtomicLong tokenCount = new AtomicLong(0); + + RangeIterator<Long, Token> memtableIterator = e.index.searchMemtable(e); + if (memtableIterator != null) + { + tokens.add(memtableIterator); + tokenCount.addAndGet(memtableIterator.getCount()); + } + + final Set<SSTableIndex> referencedIndexes = new CopyOnWriteArraySet<>(); + + try + { + final CountDownLatch latch = new CountDownLatch(perSSTableIndexes.size()); + final ExecutorService searchExecutor = SEARCH_EXECUTOR.get(); + + for (final SSTableIndex index : perSSTableIndexes) + { + if (!index.reference()) + { + latch.countDown(); + continue; + } + + // add to referenced right after the reference was acquired, + // that helps to release index if something goes bad inside of the search + referencedIndexes.add(index); + + searchExecutor.submit((Runnable) () -> { + try + { + e.checkpoint(); + + RangeIterator<Long, Token> keyIterator = index.search(e); + if (keyIterator == null) + { + releaseIndex(referencedIndexes, index); + return; + } + + tokens.add(keyIterator); + tokenCount.getAndAdd(keyIterator.getCount()); + } + catch (Throwable e1) + { + releaseIndex(referencedIndexes, index); + + if (logger.isDebugEnabled()) + logger.debug(String.format("Failed search an index %s, skipping.", index.getPath()), e1); + } + finally + { + latch.countDown(); + } + }); + } + + Uninterruptibles.awaitUninterruptibly(latch); + + // checkpoint right away after all indexes complete search because we might have crossed the quota + e.checkpoint(); + + RangeIterator<Long, Token> ranges = RangeUnionIterator.build(tokens); + return ranges == null ? null : new TermIterator(e, ranges, referencedIndexes); + } + catch (Throwable ex) + { + // if execution quota was exceeded while opening indexes or something else happened + // local (yet to be tracked) indexes should be released first before re-throwing exception + referencedIndexes.forEach(TermIterator::releaseQuietly); + + throw ex; + } + } + + protected Token computeNext() + { + try + { + return union.hasNext() ? union.next() : endOfData(); + } + finally + { + expression.checkpoint(); + } + } + + protected void performSkipTo(Long nextToken) + { + try + { + union.skipTo(nextToken); + } + finally + { + expression.checkpoint(); + } + } + + public void close() + { + FileUtils.closeQuietly(union); + referencedIndexes.forEach(TermIterator::releaseQuietly); + referencedIndexes.clear(); + } + + private static void releaseIndex(Set<SSTableIndex> indexes, SSTableIndex index) + { + indexes.remove(index); + releaseQuietly(index); + } + + private static void releaseQuietly(SSTableIndex index) + { + try + { + index.release(); + } + catch (Throwable e) + { + logger.error(String.format("Failed to release index %s", index.getPath()), e); + } + } +} http://git-wip-us.apache.org/repos/asf/cassandra/blob/72790dc8/src/java/org/apache/cassandra/index/sasi/analyzer/AbstractAnalyzer.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/index/sasi/analyzer/AbstractAnalyzer.java b/src/java/org/apache/cassandra/index/sasi/analyzer/AbstractAnalyzer.java new file mode 100644 index 0000000..b3fdd8c --- /dev/null +++ b/src/java/org/apache/cassandra/index/sasi/analyzer/AbstractAnalyzer.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.index.sasi.analyzer; + +import java.nio.ByteBuffer; +import java.text.Normalizer; +import java.util.Iterator; +import java.util.Map; + +import org.apache.cassandra.db.marshal.AbstractType; + +public abstract class AbstractAnalyzer implements Iterator<ByteBuffer> +{ + protected ByteBuffer next = null; + + public ByteBuffer next() + { + return next; + } + + public void remove() + { + throw new UnsupportedOperationException(); + } + + public abstract void init(Map<String, String> options, AbstractType validator); + + public abstract void reset(ByteBuffer input); + + public static String normalize(String original) + { + return Normalizer.isNormalized(original, Normalizer.Form.NFC) + ? original + : Normalizer.normalize(original, Normalizer.Form.NFC); + } +} http://git-wip-us.apache.org/repos/asf/cassandra/blob/72790dc8/src/java/org/apache/cassandra/index/sasi/analyzer/NoOpAnalyzer.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/index/sasi/analyzer/NoOpAnalyzer.java b/src/java/org/apache/cassandra/index/sasi/analyzer/NoOpAnalyzer.java new file mode 100644 index 0000000..9939a13 --- /dev/null +++ b/src/java/org/apache/cassandra/index/sasi/analyzer/NoOpAnalyzer.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.index.sasi.analyzer; + +import java.nio.ByteBuffer; +import java.util.Map; + +import org.apache.cassandra.db.marshal.AbstractType; + +/** + * Default noOp tokenizer. The iterator will iterate only once + * returning the unmodified input + */ +public class NoOpAnalyzer extends AbstractAnalyzer +{ + private ByteBuffer input; + private boolean hasNext = false; + + public void init(Map<String, String> options, AbstractType validator) + {} + + public boolean hasNext() + { + if (hasNext) + { + this.next = input; + this.hasNext = false; + return true; + } + return false; + } + + public void reset(ByteBuffer input) + { + this.next = null; + this.input = input; + this.hasNext = true; + } +} http://git-wip-us.apache.org/repos/asf/cassandra/blob/72790dc8/src/java/org/apache/cassandra/index/sasi/analyzer/NonTokenizingAnalyzer.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/index/sasi/analyzer/NonTokenizingAnalyzer.java b/src/java/org/apache/cassandra/index/sasi/analyzer/NonTokenizingAnalyzer.java new file mode 100644 index 0000000..676b304 --- /dev/null +++ b/src/java/org/apache/cassandra/index/sasi/analyzer/NonTokenizingAnalyzer.java @@ -0,0 +1,126 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.index.sasi.analyzer; + +import java.nio.ByteBuffer; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +import org.apache.cassandra.index.sasi.analyzer.filter.BasicResultFilters; +import org.apache.cassandra.index.sasi.analyzer.filter.FilterPipelineBuilder; +import org.apache.cassandra.index.sasi.analyzer.filter.FilterPipelineExecutor; +import org.apache.cassandra.index.sasi.analyzer.filter.FilterPipelineTask; +import org.apache.cassandra.db.marshal.AbstractType; +import org.apache.cassandra.db.marshal.AsciiType; +import org.apache.cassandra.db.marshal.UTF8Type; +import org.apache.cassandra.serializers.MarshalException; +import org.apache.cassandra.utils.ByteBufferUtil; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Analyzer that does *not* tokenize the input. Optionally will + * apply filters for the input output as defined in analyzers options + */ +public class NonTokenizingAnalyzer extends AbstractAnalyzer +{ + private static final Logger logger = LoggerFactory.getLogger(NonTokenizingAnalyzer.class); + + private static final Set<AbstractType<?>> VALID_ANALYZABLE_TYPES = new HashSet<AbstractType<?>>() + {{ + add(UTF8Type.instance); + add(AsciiType.instance); + }}; + + private AbstractType validator; + private NonTokenizingOptions options; + private FilterPipelineTask filterPipeline; + + private ByteBuffer input; + private boolean hasNext = false; + + public void init(Map<String, String> options, AbstractType validator) + { + init(NonTokenizingOptions.buildFromMap(options), validator); + } + + public void init(NonTokenizingOptions tokenizerOptions, AbstractType validator) + { + this.validator = validator; + this.options = tokenizerOptions; + this.filterPipeline = getFilterPipeline(); + } + + public boolean hasNext() + { + // check that we know how to handle the input, otherwise bail + if (!VALID_ANALYZABLE_TYPES.contains(validator)) + return false; + + if (hasNext) + { + String inputStr; + + try + { + inputStr = validator.getString(input); + if (inputStr == null) + throw new MarshalException(String.format("'null' deserialized value for %s with %s", ByteBufferUtil.bytesToHex(input), validator)); + + Object pipelineRes = FilterPipelineExecutor.execute(filterPipeline, inputStr); + if (pipelineRes == null) + return false; + + next = validator.fromString(normalize((String) pipelineRes)); + return true; + } + catch (MarshalException e) + { + logger.error("Failed to deserialize value with " + validator, e); + return false; + } + finally + { + hasNext = false; + } + } + + return false; + } + + public void reset(ByteBuffer input) + { + this.next = null; + this.input = input; + this.hasNext = true; + } + + private FilterPipelineTask getFilterPipeline() + { + FilterPipelineBuilder builder = new FilterPipelineBuilder(new BasicResultFilters.NoOperation()); + if (options.isCaseSensitive() && options.shouldLowerCaseOutput()) + builder = builder.add("to_lower", new BasicResultFilters.LowerCase()); + if (options.isCaseSensitive() && options.shouldUpperCaseOutput()) + builder = builder.add("to_upper", new BasicResultFilters.UpperCase()); + if (!options.isCaseSensitive()) + builder = builder.add("to_lower", new BasicResultFilters.LowerCase()); + return builder.build(); + } +}
