Use CQL type names in schema metadata tables patch by Aleksey Yeschenko; reviewed by Sylvain Lebresne for CASSANDRA-10365
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/340df43f Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/340df43f Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/340df43f Branch: refs/heads/trunk Commit: 340df43fb74f7f3ef021d10ad1b4510636ee3f14 Parents: f4af154 Author: Aleksey Yeschenko <alek...@apache.org> Authored: Mon Sep 21 12:02:53 2015 -0700 Committer: Aleksey Yeschenko <alek...@apache.org> Committed: Wed Nov 4 17:06:07 2015 +0000 ---------------------------------------------------------------------- CHANGES.txt | 1 + NOTICE.txt | 4 + build.xml | 9 +- ...assandra-driver-core-3.0.0-alpha4-shaded.jar | Bin 2275541 -> 0 bytes ...core-3.0.0-beta1-92c4c80-SNAPSHOT-shaded.jar | Bin 0 -> 2284484 bytes ...iver-internal-only-3.0.0a2.post0-95c6008.zip | Bin 233564 -> 0 bytes ...iver-internal-only-3.0.0a3.post0-a983923.zip | Bin 0 -> 229492 bytes lib/jgrapht-core-0.9.1.jar | Bin 0 -> 351208 bytes lib/licenses/jgrapht-core-0.9.1.txt | 227 +++ .../org/apache/cassandra/config/CFMetaData.java | 12 +- .../org/apache/cassandra/config/Schema.java | 44 +- .../org/apache/cassandra/cql3/CQL3Type.java | 77 +- .../apache/cassandra/cql3/ColumnIdentifier.java | 2 +- .../cql3/statements/CreateTableStatement.java | 12 +- .../db/DefinitionsUpdateVerbHandler.java | 3 +- src/java/org/apache/cassandra/db/Keyspace.java | 1 - .../apache/cassandra/db/marshal/EmptyType.java | 7 + .../apache/cassandra/db/marshal/TypeParser.java | 45 - .../cassandra/io/sstable/CQLSSTableWriter.java | 14 +- .../apache/cassandra/schema/CQLTypeParser.java | 93 ++ .../org/apache/cassandra/schema/Keyspaces.java | 124 ++ .../apache/cassandra/schema/SchemaKeyspace.java | 1421 +++++++----------- .../org/apache/cassandra/schema/Tables.java | 7 + src/java/org/apache/cassandra/schema/Types.java | 135 +- src/java/org/apache/cassandra/schema/Views.java | 7 + .../cassandra/service/MigrationManager.java | 15 +- .../apache/cassandra/service/MigrationTask.java | 4 - .../apache/cassandra/config/CFMetaDataTest.java | 29 +- .../cql3/validation/entities/TupleTypeTest.java | 4 +- .../cql3/validation/entities/UFTest.java | 4 +- .../schema/LegacySchemaMigratorTest.java | 12 +- .../cassandra/schema/SchemaKeyspaceTest.java | 30 +- 32 files changed, 1318 insertions(+), 1025 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cassandra/blob/340df43f/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index eb575e8..1914fa1 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,4 +1,5 @@ 3.0 + * Use CQL type names in schema metadata tables (CASSANDRA-10365) * Guard batchlog replay against integer division by zero (CASSANDRA-9223) * Fix bug when adding a column to thrift with the same name than a primary key (CASSANDRA-10608) * Add client address argument to IAuthenticator::newSaslNegotiator (CASSANDRA-8068) http://git-wip-us.apache.org/repos/asf/cassandra/blob/340df43f/NOTICE.txt ---------------------------------------------------------------------- diff --git a/NOTICE.txt b/NOTICE.txt index a20994f..b880183 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -83,3 +83,7 @@ BSD 3-clause ASM (http://asm.ow2.org/) Copyright (c) 2000-2011 INRIA, France Telecom + +JGraphT +(http://jgrapht.org) +Copyright 2003-2015, by Barak Naveh and Contributors. http://git-wip-us.apache.org/repos/asf/cassandra/blob/340df43f/build.xml ---------------------------------------------------------------------- diff --git a/build.xml b/build.xml index c8707cd..261d0e6 100644 --- a/build.xml +++ b/build.xml @@ -406,6 +406,7 @@ <dependency groupId="org.mindrot" artifactId="jbcrypt" version="0.3m" /> <dependency groupId="io.airlift" artifactId="airline" version="0.6" /> <dependency groupId="io.netty" artifactId="netty-all" version="4.0.23.Final" /> + <dependency groupId="org.jgrapht" artifactId="jgrapht-core" version="0.9.1" /> <dependency groupId="com.google.code.findbugs" artifactId="jsr305" version="2.0.2" /> <dependency groupId="com.clearspring.analytics" artifactId="stream" version="2.5.2" /> <!-- TODO CASSANDRA-9543 @@ -561,16 +562,14 @@ <!-- don't need jna to run, but nice to have --> <dependency groupId="net.java.dev.jna" artifactId="jna"/> - + <!-- don't need jamm unless running a server in which case it needs to be a -javagent to be used anyway --> <dependency groupId="com.github.jbellis" artifactId="jamm"/> <dependency groupId="io.netty" artifactId="netty-all"/> - - <dependency groupId="joda-time" artifactId="joda-time"/> - + <dependency groupId="org.jgrapht" artifactId="jgrapht-core"/> + <dependency groupId="joda-time" artifactId="joda-time"/> <dependency groupId="org.fusesource" artifactId="sigar"/> - <dependency groupId="org.eclipse.jdt.core.compiler" artifactId="ecj" /> </artifact:pom> <artifact:pom id="thrift-pom" http://git-wip-us.apache.org/repos/asf/cassandra/blob/340df43f/lib/cassandra-driver-core-3.0.0-alpha4-shaded.jar ---------------------------------------------------------------------- diff --git a/lib/cassandra-driver-core-3.0.0-alpha4-shaded.jar b/lib/cassandra-driver-core-3.0.0-alpha4-shaded.jar deleted file mode 100644 index 9a4921e..0000000 Binary files a/lib/cassandra-driver-core-3.0.0-alpha4-shaded.jar and /dev/null differ http://git-wip-us.apache.org/repos/asf/cassandra/blob/340df43f/lib/cassandra-driver-core-3.0.0-beta1-92c4c80-SNAPSHOT-shaded.jar ---------------------------------------------------------------------- diff --git a/lib/cassandra-driver-core-3.0.0-beta1-92c4c80-SNAPSHOT-shaded.jar b/lib/cassandra-driver-core-3.0.0-beta1-92c4c80-SNAPSHOT-shaded.jar new file mode 100644 index 0000000..2026c52 Binary files /dev/null and b/lib/cassandra-driver-core-3.0.0-beta1-92c4c80-SNAPSHOT-shaded.jar differ http://git-wip-us.apache.org/repos/asf/cassandra/blob/340df43f/lib/cassandra-driver-internal-only-3.0.0a2.post0-95c6008.zip ---------------------------------------------------------------------- diff --git a/lib/cassandra-driver-internal-only-3.0.0a2.post0-95c6008.zip b/lib/cassandra-driver-internal-only-3.0.0a2.post0-95c6008.zip deleted file mode 100644 index da7fa0d..0000000 Binary files a/lib/cassandra-driver-internal-only-3.0.0a2.post0-95c6008.zip and /dev/null differ http://git-wip-us.apache.org/repos/asf/cassandra/blob/340df43f/lib/cassandra-driver-internal-only-3.0.0a3.post0-a983923.zip ---------------------------------------------------------------------- diff --git a/lib/cassandra-driver-internal-only-3.0.0a3.post0-a983923.zip b/lib/cassandra-driver-internal-only-3.0.0a3.post0-a983923.zip new file mode 100644 index 0000000..66ec36e Binary files /dev/null and b/lib/cassandra-driver-internal-only-3.0.0a3.post0-a983923.zip differ http://git-wip-us.apache.org/repos/asf/cassandra/blob/340df43f/lib/jgrapht-core-0.9.1.jar ---------------------------------------------------------------------- diff --git a/lib/jgrapht-core-0.9.1.jar b/lib/jgrapht-core-0.9.1.jar new file mode 100644 index 0000000..f491e25 Binary files /dev/null and b/lib/jgrapht-core-0.9.1.jar differ http://git-wip-us.apache.org/repos/asf/cassandra/blob/340df43f/lib/licenses/jgrapht-core-0.9.1.txt ---------------------------------------------------------------------- diff --git a/lib/licenses/jgrapht-core-0.9.1.txt b/lib/licenses/jgrapht-core-0.9.1.txt new file mode 100644 index 0000000..5d80026 --- /dev/null +++ b/lib/licenses/jgrapht-core-0.9.1.txt @@ -0,0 +1,227 @@ +Eclipse Public License - v 1.0 + + THE ACCOMPANYING PROGRAM IS PROVIDED UNDER THE TERMS OF THIS ECLIPSE + PUBLIC LICENSE ("AGREEMENT"). ANY USE, REPRODUCTION OR DISTRIBUTION OF + THE PROGRAM CONSTITUTES RECIPIENT'S ACCEPTANCE OF THIS AGREEMENT. + + 1. DEFINITIONS + + "Contribution" means: + + a) in the case of the initial Contributor, the initial code and + documentation distributed under this Agreement, and + + b) in the case of each subsequent Contributor: + + i) changes to the Program, and + + ii) additions to the Program; + + where such changes and/or additions to the Program originate from and + are distributed by that particular Contributor. A Contribution + 'originates' from a Contributor if it was added to the Program by such + Contributor itself or anyone acting on such Contributor's behalf. + Contributions do not include additions to the Program which: (i) are + separate modules of software distributed in conjunction with the + Program under their own license agreement, and (ii) are not derivative + works of the Program. + + "Contributor" means any person or entity that distributes the Program. + + "Licensed Patents" mean patent claims licensable by a Contributor which + are necessarily infringed by the use or sale of its Contribution alone + or when combined with the Program. + + "Program" means the Contributions distributed in accordance with this + Agreement. + + "Recipient" means anyone who receives the Program under this Agreement, + including all Contributors. + + 2. GRANT OF RIGHTS + + a) Subject to the terms of this Agreement, each Contributor hereby + grants Recipient a non-exclusive, worldwide, royalty-free copyright + license to reproduce, prepare derivative works of, publicly display, + publicly perform, distribute and sublicense the Contribution of such + Contributor, if any, and such derivative works, in source code and + object code form. + + b) Subject to the terms of this Agreement, each Contributor hereby + grants Recipient a non-exclusive, worldwide, royalty-free patent + license under Licensed Patents to make, use, sell, offer to sell, + import and otherwise transfer the Contribution of such Contributor, if + any, in source code and object code form. This patent license shall + apply to the combination of the Contribution and the Program if, at the + time the Contribution is added by the Contributor, such addition of the + Contribution causes such combination to be covered by the Licensed + Patents. The patent license shall not apply to any other combinations + which include the Contribution. No hardware per se is licensed + hereunder. + + c) Recipient understands that although each Contributor grants the + licenses to its Contributions set forth herein, no assurances are + provided by any Contributor that the Program does not infringe the + patent or other intellectual property rights of any other entity. Each + Contributor disclaims any liability to Recipient for claims brought by + any other entity based on infringement of intellectual property rights + or otherwise. As a condition to exercising the rights and licenses + granted hereunder, each Recipient hereby assumes sole responsibility to + secure any other intellectual property rights needed, if any. For + example, if a third party patent license is required to allow Recipient + to distribute the Program, it is Recipient's responsibility to acquire + that license before distributing the Program. + + d) Each Contributor represents that to its knowledge it has sufficient + copyright rights in its Contribution, if any, to grant the copyright + license set forth in this Agreement. + + 3. REQUIREMENTS + + A Contributor may choose to distribute the Program in object code form + under its own license agreement, provided that: + + a) it complies with the terms and conditions of this Agreement; and + + b) its license agreement: + + i) effectively disclaims on behalf of all Contributors all warranties + and conditions, express and implied, including warranties or conditions + of title and non-infringement, and implied warranties or conditions of + merchantability and fitness for a particular purpose; + + ii) effectively excludes on behalf of all Contributors all liability + for damages, including direct, indirect, special, incidental and + consequential damages, such as lost profits; + + iii) states that any provisions which differ from this Agreement are + offered by that Contributor alone and not by any other party; and + + iv) states that source code for the Program is available from such + Contributor, and informs licensees how to obtain it in a reasonable + manner on or through a medium customarily used for software exchange. + + When the Program is made available in source code form: + + a) it must be made available under this Agreement; and + + b) a copy of this Agreement must be included with each copy of the + Program. + + Contributors may not remove or alter any copyright notices contained + within the Program. + + Each Contributor must identify itself as the originator of its + Contribution, if any, in a manner that reasonably allows subsequent + Recipients to identify the originator of the Contribution. + + 4. COMMERCIAL DISTRIBUTION + + Commercial distributors of software may accept certain responsibilities + with respect to end users, business partners and the like. While this + license is intended to facilitate the commercial use of the Program, + the Contributor who includes the Program in a commercial product + offering should do so in a manner which does not create potential + liability for other Contributors. Therefore, if a Contributor includes + the Program in a commercial product offering, such Contributor + ("Commercial Contributor") hereby agrees to defend and indemnify every + other Contributor ("Indemnified Contributor") against any losses, + damages and costs (collectively "Losses") arising from claims, lawsuits + and other legal actions brought by a third party against the + Indemnified Contributor to the extent caused by the acts or omissions + of such Commercial Contributor in connection with its distribution of + the Program in a commercial product offering. The obligations in this + section do not apply to any claims or Losses relating to any actual or + alleged intellectual property infringement. In order to qualify, an + Indemnified Contributor must: a) promptly notify the Commercial + Contributor in writing of such claim, and b) allow the Commercial + Contributor to control, and cooperate with the Commercial Contributor + in, the defense and any related settlement negotiations. The + Indemnified Contributor may participate in any such claim at its own + expense. + + For example, a Contributor might include the Program in a commercial + product offering, Product X. That Contributor is then a Commercial + Contributor. If that Commercial Contributor then makes performance + claims, or offers warranties related to Product X, those performance + claims and warranties are such Commercial Contributor's responsibility + alone. Under this section, the Commercial Contributor would have to + defend claims against the other Contributors related to those + performance claims and warranties, and if a court requires any other + Contributor to pay any damages as a result, the Commercial Contributor + must pay those damages. + + 5. NO WARRANTY + + EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, THE PROGRAM IS + PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, EITHER EXPRESS OR IMPLIED INCLUDING, WITHOUT LIMITATION, ANY + WARRANTIES OR CONDITIONS OF TITLE, NON-INFRINGEMENT, MERCHANTABILITY OR + FITNESS FOR A PARTICULAR PURPOSE. Each Recipient is solely responsible + for determining the appropriateness of using and distributing the + Program and assumes all risks associated with its exercise of rights + under this Agreement , including but not limited to the risks and costs + of program errors, compliance with applicable laws, damage to or loss + of data, programs or equipment, and unavailability or interruption of + operations. + + 6. DISCLAIMER OF LIABILITY + + EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, NEITHER RECIPIENT NOR + ANY CONTRIBUTORS SHALL HAVE ANY LIABILITY FOR ANY DIRECT, INDIRECT, + INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING + WITHOUT LIMITATION LOST PROFITS), HOWEVER CAUSED AND ON ANY THEORY OF + LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OR + DISTRIBUTION OF THE PROGRAM OR THE EXERCISE OF ANY RIGHTS GRANTED + HEREUNDER, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. + + 7. GENERAL + + If any provision of this Agreement is invalid or unenforceable under + applicable law, it shall not affect the validity or enforceability of + the remainder of the terms of this Agreement, and without further + action by the parties hereto, such provision shall be reformed to the + minimum extent necessary to make such provision valid and enforceable. + + If Recipient institutes patent litigation against any entity (including + a cross-claim or counterclaim in a lawsuit) alleging that the Program + itself (excluding combinations of the Program with other software or + hardware) infringes such Recipient's patent(s), then such Recipient's + rights granted under Section 2(b) shall terminate as of the date such + litigation is filed. + + All Recipient's rights under this Agreement shall terminate if it fails + to comply with any of the material terms or conditions of this + Agreement and does not cure such failure in a reasonable period of time + after becoming aware of such noncompliance. If all Recipient's rights + under this Agreement terminate, Recipient agrees to cease use and + distribution of the Program as soon as reasonably practicable. However, + Recipient's obligations under this Agreement and any licenses granted + by Recipient relating to the Program shall continue and survive. + + Everyone is permitted to copy and distribute copies of this Agreement, + but in order to avoid inconsistency the Agreement is copyrighted and + may only be modified in the following manner. The Agreement Steward + reserves the right to publish new versions (including revisions) of + this Agreement from time to time. No one other than the Agreement + Steward has the right to modify this Agreement. The Eclipse Foundation + is the initial Agreement Steward. The Eclipse Foundation may assign the + responsibility to serve as the Agreement Steward to a suitable separate + entity. Each new version of the Agreement will be given a + distinguishing version number. The Program (including Contributions) + may always be distributed subject to the version of the Agreement under + which it was received. In addition, after a new version of the + Agreement is published, Contributor may elect to distribute the Program + (including its Contributions) under the new version. Except as + expressly stated in Sections 2(a) and 2(b) above, Recipient receives no + rights or licenses to the intellectual property of any Contributor + under this Agreement, whether expressly, by implication, estoppel or + otherwise. All rights in the Program not expressly granted under this + Agreement are reserved. + + This Agreement is governed by the laws of the State of New York and the + intellectual property laws of the United States of America. No party to + this Agreement will bring a legal action under this Agreement more than + one year after the cause of action arose. Each party waives its rights + to a jury trial in any resulting litigation. http://git-wip-us.apache.org/repos/asf/cassandra/blob/340df43f/src/java/org/apache/cassandra/config/CFMetaData.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/config/CFMetaData.java b/src/java/org/apache/cassandra/config/CFMetaData.java index 0387060..86f78eb 100644 --- a/src/java/org/apache/cassandra/config/CFMetaData.java +++ b/src/java/org/apache/cassandra/config/CFMetaData.java @@ -401,7 +401,7 @@ public final class CFMetaData { CFStatement parsed = (CFStatement)QueryProcessor.parseStatement(cql); parsed.prepareKeyspace(keyspace); - CreateTableStatement statement = (CreateTableStatement) parsed.prepare().statement; + CreateTableStatement statement = (CreateTableStatement) ((CreateTableStatement.RawStatement) parsed).prepare(Types.none()).statement; return statement.metadataBuilder() .withId(generateLegacyCfId(keyspace, statement.columnFamily())) @@ -721,16 +721,6 @@ public final class CFMetaData } /** - * Updates this object in place to match the definition in the system schema tables. - * @return true if any columns were added, removed, or altered; otherwise, false is returned - */ - public boolean reload() - { - return apply(isView ? SchemaKeyspace.createViewFromName(ksName, cfName).metadata - : SchemaKeyspace.createTableFromName(ksName, cfName)); - } - - /** * Updates CFMetaData in-place to match cfm * * @return true if any columns were added, removed, or altered; otherwise, false is returned http://git-wip-us.apache.org/repos/asf/cassandra/blob/340df43f/src/java/org/apache/cassandra/config/Schema.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/config/Schema.java b/src/java/org/apache/cassandra/config/Schema.java index df4e984..117c5cd 100644 --- a/src/java/org/apache/cassandra/config/Schema.java +++ b/src/java/org/apache/cassandra/config/Schema.java @@ -122,7 +122,7 @@ public class Schema */ public Schema loadFromDisk(boolean updateVersion) { - load(SchemaKeyspace.readSchemaFromSystemTables()); + load(SchemaKeyspace.fetchNonSystemKeyspaces()); if (updateVersion) updateVersion(); return this; @@ -135,7 +135,7 @@ public class Schema * * @return self to support chaining calls */ - public Schema load(Collection<KeyspaceMetadata> keyspaceDefs) + public Schema load(Iterable<KeyspaceMetadata> keyspaceDefs) { keyspaceDefs.forEach(this::load); return this; @@ -354,6 +354,16 @@ public class Schema return keyspaces.keySet(); } + public Keyspaces getKeyspaces(Set<String> includedKeyspaceNames) + { + Keyspaces.Builder builder = Keyspaces.builder(); + keyspaces.values() + .stream() + .filter(k -> includedKeyspaceNames.contains(k.name)) + .forEach(builder::add); + return builder.build(); + } + /** * Update (or insert) new keyspace definition * @@ -611,15 +621,15 @@ public class Schema MigrationManager.instance.notifyCreateColumnFamily(cfm); } - public void updateTable(String ksName, String tableName) + public void updateTable(CFMetaData table) { - CFMetaData cfm = getCFMetaData(ksName, tableName); - assert cfm != null; - boolean columnsDidChange = cfm.reload(); + CFMetaData current = getCFMetaData(table.ksName, table.cfName); + assert current != null; + boolean columnsDidChange = current.apply(table); - Keyspace keyspace = Keyspace.open(cfm.ksName); - keyspace.getColumnFamilyStore(cfm.cfName).reload(); - MigrationManager.instance.notifyUpdateColumnFamily(cfm, columnsDidChange); + Keyspace keyspace = Keyspace.open(current.ksName); + keyspace.getColumnFamilyStore(current.cfName).reload(); + MigrationManager.instance.notifyUpdateColumnFamily(current, columnsDidChange); } public void dropTable(String ksName, String tableName) @@ -669,17 +679,15 @@ public class Schema MigrationManager.instance.notifyCreateView(view); } - public void updateView(String ksName, String viewName) + public void updateView(ViewDefinition view) { - Optional<ViewDefinition> optView = getKSMetaData(ksName).views.get(viewName); - assert optView.isPresent(); - ViewDefinition view = optView.get(); - boolean columnsDidChange = view.metadata.reload(); + ViewDefinition current = getKSMetaData(view.ksName).views.get(view.viewName).get(); + boolean columnsDidChange = current.metadata.apply(view.metadata); - Keyspace keyspace = Keyspace.open(view.ksName); - keyspace.getColumnFamilyStore(view.viewName).reload(); - Keyspace.open(view.ksName).viewManager.update(view.viewName); - MigrationManager.instance.notifyUpdateView(view, columnsDidChange); + Keyspace keyspace = Keyspace.open(current.ksName); + keyspace.getColumnFamilyStore(current.viewName).reload(); + Keyspace.open(current.ksName).viewManager.update(current.viewName); + MigrationManager.instance.notifyUpdateView(current, columnsDidChange); } public void dropView(String ksName, String viewName) http://git-wip-us.apache.org/repos/asf/cassandra/blob/340df43f/src/java/org/apache/cassandra/cql3/CQL3Type.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/cql3/CQL3Type.java b/src/java/org/apache/cassandra/cql3/CQL3Type.java index 749b989..7f5afa6 100644 --- a/src/java/org/apache/cassandra/cql3/CQL3Type.java +++ b/src/java/org/apache/cassandra/cql3/CQL3Type.java @@ -17,19 +17,21 @@ */ package org.apache.cassandra.cql3; +import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.List; +import java.util.Set; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.cassandra.config.Schema; -import org.apache.cassandra.db.SystemKeyspace; import org.apache.cassandra.db.marshal.*; import org.apache.cassandra.exceptions.InvalidRequestException; import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.exceptions.SyntaxException; import org.apache.cassandra.schema.KeyspaceMetadata; +import org.apache.cassandra.schema.Types; public interface CQL3Type { @@ -45,21 +47,22 @@ public interface CQL3Type BLOB (BytesType.instance), BOOLEAN (BooleanType.instance), COUNTER (CounterColumnType.instance), + DATE (SimpleDateType.instance), DECIMAL (DecimalType.instance), DOUBLE (DoubleType.instance), + EMPTY (EmptyType.instance), FLOAT (FloatType.instance), INET (InetAddressType.instance), INT (Int32Type.instance), SMALLINT (ShortType.instance), TEXT (UTF8Type.instance), + TIME (TimeType.instance), TIMESTAMP (TimestampType.instance), + TIMEUUID (TimeUUIDType.instance), TINYINT (ByteType.instance), UUID (UUIDType.instance), VARCHAR (UTF8Type.instance), - VARINT (IntegerType.instance), - TIMEUUID (TimeUUIDType.instance), - DATE (SimpleDateType.instance), - TIME (TimeType.instance); + VARINT (IntegerType.instance); private final AbstractType<?> type; @@ -243,7 +246,7 @@ public interface CQL3Type @Override public String toString() { - return name; + return "frozen<" + ColumnIdentifier.maybeQuote(name) + '>'; } } @@ -291,14 +294,14 @@ public interface CQL3Type public String toString() { StringBuilder sb = new StringBuilder(); - sb.append("tuple<"); + sb.append("frozen<tuple<"); for (int i = 0; i < type.size(); i++) { if (i > 0) sb.append(", "); sb.append(type.type(i).asCQL3Type()); } - sb.append(">"); + sb.append(">>"); return sb.toString(); } } @@ -342,7 +345,20 @@ public interface CQL3Type throw new InvalidRequestException(message); } - public abstract CQL3Type prepare(String keyspace) throws InvalidRequestException; + public CQL3Type prepare(String keyspace) + { + KeyspaceMetadata ksm = Schema.instance.getKSMetaData(keyspace); + if (ksm == null) + throw new ConfigurationException(String.format("Keyspace %s doesn't exist", keyspace)); + return prepare(keyspace, ksm.types); + } + + public abstract CQL3Type prepare(String keyspace, Types udts) throws InvalidRequestException; + + public boolean referencesUserType(String name) + { + return false; + } public static Raw from(CQL3Type type) { @@ -382,14 +398,14 @@ public interface CQL3Type private static class RawType extends Raw { - private CQL3Type type; + private final CQL3Type type; private RawType(CQL3Type type) { this.type = type; } - public CQL3Type prepare(String keyspace) throws InvalidRequestException + public CQL3Type prepare(String keyspace, Types udts) throws InvalidRequestException { return type; } @@ -443,7 +459,7 @@ public interface CQL3Type return true; } - public CQL3Type prepare(String keyspace) throws InvalidRequestException + public CQL3Type prepare(String keyspace, Types udts) throws InvalidRequestException { assert values != null : "Got null values type for a collection"; @@ -461,16 +477,21 @@ public interface CQL3Type switch (kind) { case LIST: - return new Collection(ListType.getInstance(values.prepare(keyspace).getType(), !frozen)); + return new Collection(ListType.getInstance(values.prepare(keyspace, udts).getType(), !frozen)); case SET: - return new Collection(SetType.getInstance(values.prepare(keyspace).getType(), !frozen)); + return new Collection(SetType.getInstance(values.prepare(keyspace, udts).getType(), !frozen)); case MAP: assert keys != null : "Got null keys type for a collection"; - return new Collection(MapType.getInstance(keys.prepare(keyspace).getType(), values.prepare(keyspace).getType(), !frozen)); + return new Collection(MapType.getInstance(keys.prepare(keyspace, udts).getType(), values.prepare(keyspace, udts).getType(), !frozen)); } throw new AssertionError(); } + public boolean referencesUserType(String name) + { + return (keys != null && keys.referencesUserType(name)) || values.referencesUserType(name); + } + @Override public String toString() { @@ -510,13 +531,13 @@ public interface CQL3Type return false; } - public CQL3Type prepare(String keyspace) throws InvalidRequestException + public CQL3Type prepare(String keyspace, Types udts) throws InvalidRequestException { if (name.hasKeyspace()) { // The provided keyspace is the one of the current statement this is part of. If it's different from the keyspace of // the UTName, we reject since we want to limit user types to their own keyspace (see #6643) - if (keyspace != null && !SystemKeyspace.NAME.equals(name.getKeyspace()) && !keyspace.equals(name.getKeyspace())) + if (!keyspace.equals(name.getKeyspace())) throw new InvalidRequestException(String.format("Statement on keyspace %s cannot refer to a user type in keyspace %s; " + "user types can only be used in the keyspace they are defined in", keyspace, name.getKeyspace())); @@ -526,10 +547,7 @@ public interface CQL3Type name.setKeyspace(keyspace); } - KeyspaceMetadata ksm = Schema.instance.getKSMetaData(name.getKeyspace()); - if (ksm == null) - throw new InvalidRequestException("Unknown keyspace " + name.getKeyspace()); - UserType type = ksm.types.getNullable(name.getUserTypeName()); + UserType type = udts.getNullable(name.getUserTypeName()); if (type == null) throw new InvalidRequestException("Unknown type " + name); @@ -539,6 +557,11 @@ public interface CQL3Type return new UserDefined(name.toString(), type); } + public boolean referencesUserType(String name) + { + return this.name.getStringTypeName().equals(name); + } + protected boolean supportsFreezing() { return true; @@ -573,14 +596,13 @@ public interface CQL3Type public void freeze() throws InvalidRequestException { for (CQL3Type.Raw t : types) - { if (t.supportsFreezing()) t.freeze(); - } + frozen = true; } - public CQL3Type prepare(String keyspace) throws InvalidRequestException + public CQL3Type prepare(String keyspace, Types udts) throws InvalidRequestException { if (!frozen) freeze(); @@ -591,11 +613,16 @@ public interface CQL3Type if (t.isCounter()) throw new InvalidRequestException("Counters are not allowed inside tuples"); - ts.add(t.prepare(keyspace).getType()); + ts.add(t.prepare(keyspace, udts).getType()); } return new Tuple(new TupleType(ts)); } + public boolean referencesUserType(String name) + { + return types.stream().anyMatch(t -> t.referencesUserType(name)); + } + @Override public String toString() { http://git-wip-us.apache.org/repos/asf/cassandra/blob/340df43f/src/java/org/apache/cassandra/cql3/ColumnIdentifier.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/cql3/ColumnIdentifier.java b/src/java/org/apache/cassandra/cql3/ColumnIdentifier.java index eb16f93..745e4f0 100644 --- a/src/java/org/apache/cassandra/cql3/ColumnIdentifier.java +++ b/src/java/org/apache/cassandra/cql3/ColumnIdentifier.java @@ -328,7 +328,7 @@ public class ColumnIdentifier extends org.apache.cassandra.cql3.selection.Select } } - private static String maybeQuote(String text) + static String maybeQuote(String text) { if (UNQUOTED_IDENTIFIER.matcher(text).matches()) return text; http://git-wip-us.apache.org/repos/asf/cassandra/blob/340df43f/src/java/org/apache/cassandra/cql3/statements/CreateTableStatement.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/cql3/statements/CreateTableStatement.java b/src/java/org/apache/cassandra/cql3/statements/CreateTableStatement.java index 8e1d6c6..a1947df 100644 --- a/src/java/org/apache/cassandra/cql3/statements/CreateTableStatement.java +++ b/src/java/org/apache/cassandra/cql3/statements/CreateTableStatement.java @@ -30,7 +30,9 @@ import org.apache.cassandra.cql3.*; import org.apache.cassandra.db.*; import org.apache.cassandra.db.marshal.*; import org.apache.cassandra.exceptions.*; +import org.apache.cassandra.schema.KeyspaceMetadata; import org.apache.cassandra.schema.TableParams; +import org.apache.cassandra.schema.Types; import org.apache.cassandra.service.ClientState; import org.apache.cassandra.service.MigrationManager; import org.apache.cassandra.service.QueryState; @@ -193,6 +195,14 @@ public class CreateTableStatement extends SchemaAlteringStatement */ public ParsedStatement.Prepared prepare() throws RequestValidationException { + KeyspaceMetadata ksm = Schema.instance.getKSMetaData(keyspace()); + if (ksm == null) + throw new ConfigurationException(String.format("Keyspace %s doesn't exist", keyspace())); + return prepare(ksm.types); + } + + public ParsedStatement.Prepared prepare(Types udts) throws RequestValidationException + { // Column family name if (!columnFamily().matches("\\w+")) throw new InvalidRequestException(String.format("\"%s\" is not a valid table name (must be alphanumeric character or underscore only: [a-zA-Z_0-9]+)", columnFamily())); @@ -212,7 +222,7 @@ public class CreateTableStatement extends SchemaAlteringStatement for (Map.Entry<ColumnIdentifier, CQL3Type.Raw> entry : definitions.entrySet()) { ColumnIdentifier id = entry.getKey(); - CQL3Type pt = entry.getValue().prepare(keyspace()); + CQL3Type pt = entry.getValue().prepare(keyspace(), udts); if (pt.isCollection() && ((CollectionType)pt.getType()).isMultiCell()) stmt.collections.put(id.bytes, (CollectionType)pt.getType()); if (entry.getValue().isCounter()) http://git-wip-us.apache.org/repos/asf/cassandra/blob/340df43f/src/java/org/apache/cassandra/db/DefinitionsUpdateVerbHandler.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/db/DefinitionsUpdateVerbHandler.java b/src/java/org/apache/cassandra/db/DefinitionsUpdateVerbHandler.java index b849f95..8b3e121 100644 --- a/src/java/org/apache/cassandra/db/DefinitionsUpdateVerbHandler.java +++ b/src/java/org/apache/cassandra/db/DefinitionsUpdateVerbHandler.java @@ -24,6 +24,7 @@ import org.slf4j.LoggerFactory; import org.apache.cassandra.concurrent.Stage; import org.apache.cassandra.concurrent.StageManager; +import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.net.IVerbHandler; import org.apache.cassandra.net.MessageIn; import org.apache.cassandra.schema.SchemaKeyspace; @@ -45,7 +46,7 @@ public class DefinitionsUpdateVerbHandler implements IVerbHandler<Collection<Mut StageManager.getStage(Stage.MIGRATION).submit(new WrappedRunnable() { - public void runMayThrow() throws Exception + public void runMayThrow() throws ConfigurationException { SchemaKeyspace.mergeSchemaAndAnnounceVersion(message.payload); } http://git-wip-us.apache.org/repos/asf/cassandra/blob/340df43f/src/java/org/apache/cassandra/db/Keyspace.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/db/Keyspace.java b/src/java/org/apache/cassandra/db/Keyspace.java index 293f8a3..eb28b2c 100644 --- a/src/java/org/apache/cassandra/db/Keyspace.java +++ b/src/java/org/apache/cassandra/db/Keyspace.java @@ -377,7 +377,6 @@ public class Keyspace // re-initializing an existing CF. This will happen if you cleared the schema // on this node and it's getting repopulated from the rest of the cluster. assert cfs.name.equals(cfName); - cfs.metadata.reload(); cfs.reload(); } } http://git-wip-us.apache.org/repos/asf/cassandra/blob/340df43f/src/java/org/apache/cassandra/db/marshal/EmptyType.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/db/marshal/EmptyType.java b/src/java/org/apache/cassandra/db/marshal/EmptyType.java index 9cd7226..c653084 100644 --- a/src/java/org/apache/cassandra/db/marshal/EmptyType.java +++ b/src/java/org/apache/cassandra/db/marshal/EmptyType.java @@ -19,6 +19,7 @@ package org.apache.cassandra.db.marshal; import java.nio.ByteBuffer; +import org.apache.cassandra.cql3.CQL3Type; import org.apache.cassandra.cql3.Constants; import org.apache.cassandra.cql3.Term; import org.apache.cassandra.serializers.TypeSerializer; @@ -65,6 +66,12 @@ public class EmptyType extends AbstractType<Void> return new Constants.Value(ByteBufferUtil.EMPTY_BYTE_BUFFER); } + @Override + public CQL3Type asCQL3Type() + { + return CQL3Type.Native.EMPTY; + } + public TypeSerializer<Void> getSerializer() { return EmptySerializer.instance; http://git-wip-us.apache.org/repos/asf/cassandra/blob/340df43f/src/java/org/apache/cassandra/db/marshal/TypeParser.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/db/marshal/TypeParser.java b/src/java/org/apache/cassandra/db/marshal/TypeParser.java index faa678e..35d15ab 100644 --- a/src/java/org/apache/cassandra/db/marshal/TypeParser.java +++ b/src/java/org/apache/cassandra/db/marshal/TypeParser.java @@ -23,7 +23,6 @@ import java.lang.reflect.Method; import java.nio.ByteBuffer; import java.util.*; -import org.apache.cassandra.cql3.CQL3Type; import org.apache.cassandra.exceptions.*; import org.apache.cassandra.utils.ByteBufferUtil; import org.apache.cassandra.utils.FBUtilities; @@ -94,50 +93,6 @@ public class TypeParser return parse(compareWith == null ? null : compareWith.toString()); } - public static String parseCqlNativeType(String str) - { - return CQL3Type.Native.valueOf(str.trim().toUpperCase(Locale.ENGLISH)).getType().toString(); - } - - public static String parseCqlCollectionOrFrozenType(String str) throws SyntaxException - { - str = str.trim().toLowerCase(); - switch (str) - { - case "map": return "MapType"; - case "set": return "SetType"; - case "list": return "ListType"; - case "frozen": return "FrozenType"; - default: throw new SyntaxException("Invalid type name" + str); - } - } - - /** - * Turns user facing type names into Abstract Types, 'text' -> UTF8Type - */ - public static AbstractType<?> parseCqlName(String str) throws SyntaxException, ConfigurationException - { - return parse(parseCqlNameRecurse(str)); - } - - private static String parseCqlNameRecurse(String str) throws SyntaxException - { - if (str.indexOf(',') >= 0 && (!str.contains("<") || (str.indexOf(',') < str.indexOf('<')))) - { - String[] parseString = str.split(",", 2); - return parseCqlNameRecurse(parseString[0]) + "," + parseCqlNameRecurse(parseString[1]); - } - else if (str.contains("<")) - { - String[] parseString = str.trim().split("<", 2); - return parseCqlCollectionOrFrozenType(parseString[0]) + "(" + parseCqlNameRecurse(parseString[1].substring(0, parseString[1].length()-1)) + ")"; - } - else - { - return parseCqlNativeType(str); - } - } - /** * Parse an AbstractType from current position of this parser. */ http://git-wip-us.apache.org/repos/asf/cassandra/blob/340df43f/src/java/org/apache/cassandra/io/sstable/CQLSSTableWriter.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/io/sstable/CQLSSTableWriter.java b/src/java/org/apache/cassandra/io/sstable/CQLSSTableWriter.java index 70380f4..0006a81 100644 --- a/src/java/org/apache/cassandra/io/sstable/CQLSSTableWriter.java +++ b/src/java/org/apache/cassandra/io/sstable/CQLSSTableWriter.java @@ -25,6 +25,7 @@ import java.util.*; import org.apache.cassandra.config.*; import org.apache.cassandra.cql3.*; +import org.apache.cassandra.cql3.statements.CFStatement; import org.apache.cassandra.cql3.statements.CreateTableStatement; import org.apache.cassandra.cql3.statements.ParsedStatement; import org.apache.cassandra.cql3.statements.UpdateStatement; @@ -40,6 +41,7 @@ import org.apache.cassandra.io.sstable.format.SSTableFormat; import org.apache.cassandra.schema.KeyspaceMetadata; import org.apache.cassandra.schema.KeyspaceParams; import org.apache.cassandra.schema.Tables; +import org.apache.cassandra.schema.Types; import org.apache.cassandra.service.ClientState; import org.apache.cassandra.utils.Pair; @@ -348,7 +350,7 @@ public class CQLSSTableWriter implements Closeable { synchronized (CQLSSTableWriter.class) { - this.schema = getStatement(schema, CreateTableStatement.class, "CREATE TABLE").left.getCFMetaData(); + this.schema = getTableMetadata(schema); // We need to register the keyspace/table metadata through Schema, otherwise we won't be able to properly // build the insert statement in using(). @@ -482,6 +484,16 @@ public class CQLSSTableWriter implements Closeable return this; } + private static CFMetaData getTableMetadata(String schema) + { + CFStatement parsed = (CFStatement)QueryProcessor.parseStatement(schema); + // tables with UDTs are currently not supported by CQLSSTableWrite, so we just use Types.none(), for now + // see CASSANDRA-10624 for more details + CreateTableStatement statement = (CreateTableStatement) ((CreateTableStatement.RawStatement) parsed).prepare(Types.none()).statement; + statement.validate(ClientState.forInternalCalls()); + return statement.getCFMetaData(); + } + private static <T extends CQLStatement> Pair<T, List<ColumnSpecification>> getStatement(String query, Class<T> klass, String type) { try http://git-wip-us.apache.org/repos/asf/cassandra/blob/340df43f/src/java/org/apache/cassandra/schema/CQLTypeParser.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/schema/CQLTypeParser.java b/src/java/org/apache/cassandra/schema/CQLTypeParser.java new file mode 100644 index 0000000..87eebd7 --- /dev/null +++ b/src/java/org/apache/cassandra/schema/CQLTypeParser.java @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.schema; + +import com.google.common.collect.ImmutableSet; + +import org.antlr.runtime.*; +import org.apache.cassandra.cql3.*; +import org.apache.cassandra.db.marshal.AbstractType; +import org.apache.cassandra.db.marshal.UserType; +import org.apache.cassandra.exceptions.SyntaxException; + +import static org.apache.cassandra.utils.ByteBufferUtil.bytes; + +public final class CQLTypeParser +{ + private static final ImmutableSet<String> PRIMITIVE_TYPES; + + static + { + ImmutableSet.Builder<String> builder = ImmutableSet.builder(); + for (CQL3Type.Native primitive : CQL3Type.Native.values()) + builder.add(primitive.name().toLowerCase()); + PRIMITIVE_TYPES = builder.build(); + } + + public static AbstractType<?> parse(String keyspace, String unparsed, Types userTypes) + { + String lowercased = unparsed.toLowerCase(); + + // fast path for the common case of a primitive type + if (PRIMITIVE_TYPES.contains(lowercased)) + return CQL3Type.Native.valueOf(unparsed.toUpperCase()).getType(); + + // special-case top-level UDTs + UserType udt = userTypes.getNullable(bytes(lowercased)); + if (udt != null) + return udt; + + return parseRaw(unparsed).prepare(keyspace, userTypes).getType(); + } + + static CQL3Type.Raw parseRaw(String type) + { + try + { + // Lexer and parser + ErrorCollector errorCollector = new ErrorCollector(type); + CharStream stream = new ANTLRStringStream(type); + CqlLexer lexer = new CqlLexer(stream); + lexer.addErrorListener(errorCollector); + + TokenStream tokenStream = new CommonTokenStream(lexer); + CqlParser parser = new CqlParser(tokenStream); + parser.addErrorListener(errorCollector); + + // Parse the query string to a statement instance + CQL3Type.Raw rawType = parser.comparatorType(); + + // The errorCollector has queue up any errors that the lexer and parser may have encountered + // along the way, if necessary, we turn the last error into exceptions here. + errorCollector.throwFirstSyntaxError(); + + return rawType; + } + catch (RuntimeException re) + { + throw new SyntaxException(String.format("Failed parsing statement: [%s] reason: %s %s", + type, + re.getClass().getSimpleName(), + re.getMessage())); + } + catch (RecognitionException e) + { + throw new SyntaxException("Invalid or malformed CQL type: " + e.getMessage()); + } + } +} http://git-wip-us.apache.org/repos/asf/cassandra/blob/340df43f/src/java/org/apache/cassandra/schema/Keyspaces.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/schema/Keyspaces.java b/src/java/org/apache/cassandra/schema/Keyspaces.java new file mode 100644 index 0000000..8c0a63e --- /dev/null +++ b/src/java/org/apache/cassandra/schema/Keyspaces.java @@ -0,0 +1,124 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.schema; + +import java.util.Iterator; +import java.util.function.Predicate; +import java.util.stream.Stream; + +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.MapDifference; +import com.google.common.collect.Maps; + +public final class Keyspaces implements Iterable<KeyspaceMetadata> +{ + private final ImmutableMap<String, KeyspaceMetadata> keyspaces; + + private Keyspaces(Builder builder) + { + keyspaces = builder.keyspaces.build(); + } + + public static Builder builder() + { + return new Builder(); + } + + public static Keyspaces none() + { + return builder().build(); + } + + public static Keyspaces of(KeyspaceMetadata... keyspaces) + { + return builder().add(keyspaces).build(); + } + + public Iterator<KeyspaceMetadata> iterator() + { + return keyspaces.values().iterator(); + } + + public Stream<KeyspaceMetadata> stream() + { + return keyspaces.values().stream(); + } + + public Keyspaces filter(Predicate<KeyspaceMetadata> predicate) + { + Builder builder = builder(); + stream().filter(predicate).forEach(builder::add); + return builder.build(); + } + + MapDifference<String, KeyspaceMetadata> diff(Keyspaces other) + { + return Maps.difference(keyspaces, other.keyspaces); + } + + @Override + public boolean equals(Object o) + { + return this == o || (o instanceof Keyspaces && keyspaces.equals(((Keyspaces) o).keyspaces)); + } + + @Override + public int hashCode() + { + return keyspaces.hashCode(); + } + + @Override + public String toString() + { + return keyspaces.values().toString(); + } + + public static final class Builder + { + private final ImmutableMap.Builder<String, KeyspaceMetadata> keyspaces = new ImmutableMap.Builder<>(); + + private Builder() + { + } + + public Keyspaces build() + { + return new Keyspaces(this); + } + + public Builder add(KeyspaceMetadata keyspace) + { + keyspaces.put(keyspace.name, keyspace); + return this; + } + + public Builder add(KeyspaceMetadata... keyspaces) + { + for (KeyspaceMetadata keyspace : keyspaces) + add(keyspace); + return this; + } + + public Builder add(Iterable<KeyspaceMetadata> keyspaces) + { + keyspaces.forEach(this::add); + return this; + } + } +}