Updated Branches: refs/heads/trunk 4b08be49a -> e5199eeac
User types support for cqlsh. Patch by Mikhail Stepura; reviewed by Aleksey Yeschenko for CASSANDRA-6305 Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/e5199eea Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/e5199eea Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/e5199eea Branch: refs/heads/trunk Commit: e5199eeac1e5a5cb1c402e146955ae26cf595f96 Parents: 4b08be4 Author: Mikhail Stepura <[email protected]> Authored: Mon Jan 6 18:51:50 2014 -0800 Committer: Mikhail Stepura <[email protected]> Committed: Fri Feb 14 10:55:32 2014 -0800 ---------------------------------------------------------------------- bin/cqlsh | 101 ++++++++++++++++++++++++++++++++-- pylib/cqlshlib/cql3handling.py | 105 +++++++++++++++++++++++++++++++++--- pylib/cqlshlib/formatting.py | 20 ++++++- pylib/cqlshlib/usertypes.py | 63 ++++++++++++++++++++++ 4 files changed, 279 insertions(+), 10 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cassandra/blob/e5199eea/bin/cqlsh ---------------------------------------------------------------------- diff --git a/bin/cqlsh b/bin/cqlsh index 0add46c..427a0a2 100755 --- a/bin/cqlsh +++ b/bin/cqlsh @@ -32,7 +32,7 @@ exit 1 from __future__ import with_statement description = "CQL Shell for Apache Cassandra" -version = "4.1.1" +version = "4.1.2" from StringIO import StringIO from itertools import groupby @@ -112,7 +112,7 @@ cqlshlibdir = os.path.join(CASSANDRA_PATH, 'pylib') if os.path.isdir(cqlshlibdir): sys.path.insert(0, cqlshlibdir) -from cqlshlib import cqlhandling, cql3handling, pylexotron +from cqlshlib import cqlhandling, cql3handling, pylexotron, usertypes from cqlshlib.displaying import (RED, BLUE, ANSI_RESET, COLUMN_NAME_COLORS, FormattedValue, colorme) from cqlshlib.formatting import format_by_type @@ -230,7 +230,9 @@ cqlsh_extra_syntax_rules = r''' | ( "COLUMNFAMILY" | "TABLE" ) cf=<columnFamilyName> | ( "COLUMNFAMILIES" | "TABLES" ) | "FULL"? "SCHEMA" - | "CLUSTER" ) + | "CLUSTER" + | "TYPES" + | "TYPE" ut=<userTypeName>) ; <consistencyCommand> ::= "CONSISTENCY" ( level=<consistencyLevel> )? @@ -366,6 +368,9 @@ class ColumnFamilyNotFound(Exception): class VersionNotSupported(Exception): pass +class UserTypeNotFound(Exception): + pass + class DecodeError(Exception): verb = 'decode' @@ -455,6 +460,7 @@ class Shell(cmd.Cmd): shunted_query_out = None csv_dialect_defaults = dict(delimiter=',', doublequote=False, escapechar='\\', quotechar='"') + utypes_meta = None def __init__(self, hostname, port, transport_factory, color=False, username=None, password=None, encoding=None, stdin=None, tty=True, @@ -514,6 +520,9 @@ class Shell(cmd.Cmd): self.show_line_nums = True self.stdin = stdin self.query_out = sys.stdout + self.refresh_utypes_meta() + #UserTypes will use it to get theirs field names + usertypes.get_field_names = lambda k, t: self.utypes_meta.get_field_names(k, t) def set_expanded_cql_version(self, ver): ver, vertuple = full_cql_version(ver) @@ -626,6 +635,21 @@ class Shell(cmd.Cmd): layout = self.get_columnfamily_layout(ksname, cfname) return [col.name for col in layout.columns] + def get_usertype_names(self, ksname=None): + if ksname is None: + ksname = self.current_keyspace + + return self.utypes_meta.get_usertypes_names(ksname) + + def get_usertype_layout(self, ksname, typename): + if ksname is None: + ksname = self.current_keyspace + layout = self.utypes_meta.get_fields_with_types(ksname, typename) + + if not layout: + raise UserTypeNotFound("User type %r not found" % typename) + return layout + # ===== thrift-dependent parts ===== def get_cluster_name(self): @@ -693,6 +717,15 @@ class Shell(cmd.Cmd): cols = self.fetchdict_all() return cql3handling.CqlTableDef.from_layout(layout, cols) + def build_usertypes_meta(self): + ut_q = """select * from system.schema_usertypes""" + self.cursor.execute(ut_q, consistency_level='ONE') + data = self.fetchdict_all() + if not data: + return None + + return cql3handling.UserTypesMeta.from_layout(data) + # ===== end cql3-dependent parts ===== def reset_statement(self): @@ -755,6 +788,15 @@ class Shell(cmd.Cmd): return yield newline + def refresh_utypes_meta(self): + try: + self.utypes_meta = self.build_usertypes_meta() + except Exception, e: + if self.debug: + import traceback + traceback.print_exc() + + def cmdloop(self): """ Adapted from cmd.Cmd's version, because there is literally no way with @@ -926,6 +968,8 @@ class Shell(cmd.Cmd): self.writeresult("") self.print_static_result(self.cursor) self.flush_output() + if self.could_change_schema(statement): + self.refresh_utypes_meta() return True def get_nametype(self, cursor, num): @@ -1082,6 +1126,14 @@ class Shell(cmd.Cmd): out.write(" AND durable_writes = 'false'") out.write(';\n') + #print usertypes first + uts = self.get_usertype_names(ksname) + if uts: + out.write('\nUSE %s;\n' % ksname) + for ut in uts: + out.write('\n') + self.print_recreate_usertype(ksdef.name, ut, out) + cfs = self.get_columnfamily_names(ksname) if cfs: out.write('\nUSE %s;\n' % ksname) @@ -1090,6 +1142,16 @@ class Shell(cmd.Cmd): # yes, cf might be looked up again. oh well. self.print_recreate_columnfamily(ksdef.name, cf, out) + def print_recreate_usertype(self, ksname, utname, out): + layout = self.get_usertype_layout(ksname, utname) + out.write("CREATE TYPE %s (\n" % utname) + for (index,(tname,ttype)) in enumerate(layout, 1): + out.write(" %s %s" % (tname, ttype)) + if index < len(layout): + out.write(",\n") + else: + out.write("\n); \n") + def print_recreate_columnfamily(self, ksname, cfname, out): """ Output CQL commands which should be pasteable back into a CQL session @@ -1099,6 +1161,7 @@ class Shell(cmd.Cmd): """ layout = self.get_columnfamily_layout(ksname, cfname) cfname = self.cql_protect_name(layout.name) + out.write("CREATE TABLE %s (\n" % cfname) for col in layout.columns: @@ -1238,6 +1301,26 @@ class Shell(cmd.Cmd): cmd.Cmd.columnize(self, self.get_columnfamily_names(ksname)) print + def describe_usertypes(self, ksname): + print + if ksname is None: + for k in self.get_keyspaces(): + name = self.cql_protect_name(k.name) + print 'Keyspace %s' % (name,) + print '---------%s' % ('-' * len(name)) + cmd.Cmd.columnize(self, self.get_usertype_names(k.name)) + print + else: + cmd.Cmd.columnize(self, self.get_usertype_names(ksname)) + print + + def describe_usertype(self, ksname, typename): + if ksname is None: + ksname = self.current_keyspace + print + self.print_recreate_usertype(ksname, typename, sys.stdout) + print + def describe_cluster(self): print '\nCluster: %s' % self.get_cluster_name() p = trim_if_present(self.get_partitioner(), 'org.apache.cassandra.dht.') @@ -1323,6 +1406,12 @@ class Shell(cmd.Cmd): self.describe_columnfamily(ks, cf) elif what in ('columnfamilies', 'tables'): self.describe_columnfamilies(self.current_keyspace) + elif what == 'types': + self.describe_usertypes(self.current_keyspace) + elif what == 'type': + ks = self.cql_unprotect_name(parsed.get_binding('ksname', None)) + ut = self.cql_unprotect_name(parsed.get_binding('utname')) + self.describe_usertype(ks, ut) elif what == 'cluster': self.describe_cluster() elif what == 'schema': @@ -1850,6 +1939,12 @@ class Shell(cmd.Cmd): text = '%s:%d:%s' % (self.stdin.name, self.lineno, text) self.writeresult(text, color, newline=newline, out=sys.stderr) + @staticmethod + def could_change_schema(statement): + import re + return re.match(r"(alter|create|drop)\s+type", statement.lower()) + + class ErrorHandlingSchemaDecoder(cql.decoders.SchemaDecoder): def name_decode_error(self, err, namebytes, expectedtype): return DecodeError(namebytes, err, expectedtype) http://git-wip-us.apache.org/repos/asf/cassandra/blob/e5199eea/pylib/cqlshlib/cql3handling.py ---------------------------------------------------------------------- diff --git a/pylib/cqlshlib/cql3handling.py b/pylib/cqlshlib/cql3handling.py index cf26e6e..093354a 100644 --- a/pylib/cqlshlib/cql3handling.py +++ b/pylib/cqlshlib/cql3handling.py @@ -245,11 +245,14 @@ JUNK ::= /([ \t\r\f\v]+|(--|[/][/])[^\n\r]*([\n\r]|$)|[/][*].*?[*][/])/ ; <schemaChangeStatement> ::= <createKeyspaceStatement> | <createColumnFamilyStatement> | <createIndexStatement> + | <createUserTypeStatement> | <dropKeyspaceStatement> | <dropColumnFamilyStatement> | <dropIndexStatement> + | <dropUserTypeStatement> | <alterTableStatement> | <alterKeyspaceStatement> + | <alterUserTypeStatement> ; <authenticationStatement> ::= <createUserStatement> @@ -266,15 +269,19 @@ JUNK ::= /([ \t\r\f\v]+|(--|[/][/])[^\n\r]*([\n\r]|$)|[/][*].*?[*][/])/ ; # timestamp is included here, since it's also a keyword <simpleStorageType> ::= typename=( <identifier> | <stringLiteral> | <K_TIMESTAMP> ) ; -<storageType> ::= <simpleStorageType> | <collectionType> ; +<userType> ::= utname=<cfOrKsName> ; -<collectionType> ::= "map" "<" <simpleStorageType> "," <simpleStorageType> ">" - | "list" "<" <simpleStorageType> ">" - | "set" "<" <simpleStorageType> ">" +<storageType> ::= <simpleStorageType> | <collectionType> | <userType> ; + +<collectionType> ::= "map" "<" <simpleStorageType> "," ( <simpleStorageType> | <userType> ) ">" + | "list" "<" ( <simpleStorageType> | <userType> ) ">" + | "set" "<" ( <simpleStorageType> | <userType> ) ">" ; <columnFamilyName> ::= ( ksname=<cfOrKsName> dot="." )? cfname=<cfOrKsName> ; +<userTypeName> ::= ( ksname=<cfOrKsName> dot="." )? utname=<cfOrKsName> ; + <keyspaceName> ::= ksname=<cfOrKsName> ; <nonSystemKeyspaceName> ::= ksname=<cfOrKsName> ; @@ -508,17 +515,19 @@ def ks_name_completer(ctxt, cass): ksnames = [n for n in cass.get_keyspace_names() if n not in NONALTERBALE_KEYSPACES] return map(maybe_escape_name, ksnames) -@completer_for('columnFamilyName', 'ksname') def cf_ks_name_completer(ctxt, cass): return [maybe_escape_name(ks) + '.' for ks in cass.get_keyspace_names()] -@completer_for('columnFamilyName', 'dot') +completer_for('columnFamilyName', 'ksname')(cf_ks_name_completer) + def cf_ks_dot_completer(ctxt, cass): name = dequote_name(ctxt.get_binding('ksname')) if name in cass.get_keyspace_names(): return ['.'] return [] +completer_for('columnFamilyName', 'dot')(cf_ks_dot_completer) + @completer_for('columnFamilyName', 'cfname') def cf_name_completer(ctxt, cass): ks = ctxt.get_binding('ksname', None) @@ -532,6 +541,26 @@ def cf_name_completer(ctxt, cass): raise return map(maybe_escape_name, cfnames) +completer_for('userTypeName', 'ksname')(cf_ks_name_completer) + +completer_for('userTypeName', 'dot')(cf_ks_dot_completer) + +def ut_name_completer(ctxt, cass): + ks = ctxt.get_binding('ksname', None) + if ks is not None: + ks = dequote_name(ks) + try: + utnames = cass.get_usertype_names(ks) + except Exception: + if ks is None: + return () + raise + return map(maybe_escape_name, utnames) + + +completer_for('userTypeName', 'utname')(ut_name_completer) +completer_for('userType', 'utname')(ut_name_completer) + @completer_for('unreservedKeyword', 'nocomplete') def unreserved_keyword_completer(ctxt, cass): # we never want to provide completions through this production; @@ -546,6 +575,13 @@ def get_cf_layout(ctxt, cass): cf = dequote_name(ctxt.get_binding('cfname')) return cass.get_columnfamily_layout(ks, cf) +def get_ut_layout(ctxt, cass): + ks = ctxt.get_binding('ksname', None) + if ks is not None: + ks = dequote_name(ks) + ut = dequote_name(ctxt.get_binding('utname')) + return cass.get_usertype_layout(ks, ut) + def working_on_keyspace(ctxt): wat = ctxt.get_binding('wat').upper() if wat in ('KEYSPACE', 'SCHEMA'): @@ -930,9 +966,16 @@ syntax_rules += r''' cf=<columnFamilyName> "(" col=<cident> ")" ( "USING" <stringLiteral> ( "WITH" "OPTIONS" "=" <mapLiteral> )? )? ; + +<createUserTypeStatement> ::= "CREATE" "TYPE" ( ks=<nonSystemKeyspaceName> dot="." )? typename=<cfOrKsName> "(" newcol=<cident> <storageType> + ( "," [newcolname]=<cident> <storageType> )* + ")" + ; ''' explain_completion('createIndexStatement', 'indexname', '<new_index_name>') +explain_completion('createUserTypeStatement', 'typename', '<new_type_name>') +explain_completion('createUserTypeStatement', 'newcol', '<new_field_name>') @completer_for('createIndexStatement', 'col') def create_index_col_completer(ctxt, cass): @@ -949,6 +992,10 @@ syntax_rules += r''' <dropIndexStatement> ::= "DROP" "INDEX" ("IF" "EXISTS")? indexname=<identifier> ; + +<dropUserTypeStatement> ::= "DROP" "TYPE" ut=<userTypeName> + ; + ''' @completer_for('dropIndexStatement', 'indexname') @@ -966,6 +1013,15 @@ syntax_rules += r''' | "RENAME" existcol=<cident> "TO" newcol=<cident> ( "AND" existcol=<cident> "TO" newcol=<cident> )* ; + +<alterUserTypeStatement> ::= "ALTER" "TYPE" ut=<userTypeName> + <alterTypeInstructions> + ; +<alterTypeInstructions> ::= "RENAME" "TO" typename=<cfOrKsName> + | "ALTER" existcol=<cident> "TYPE" <storageType> + | "ADD" newcol=<cident> <storageType> + | "RENAME" existcol=<cident> "TO" newcol=<cident> + ; ''' @completer_for('alterInstructions', 'existcol') @@ -974,7 +1030,16 @@ def alter_table_col_completer(ctxt, cass): cols = [md.name for md in layout.columns] return map(maybe_escape_name, cols) +@completer_for('alterTypeInstructions', 'existcol') +def alter_type_field_completer(ctxt, cass): + layout = get_ut_layout(ctxt, cass) + fields = [tuple[0] for tuple in layout] + return map(maybe_escape_name, fields) + explain_completion('alterInstructions', 'newcol', '<new_column_name>') +explain_completion('alterTypeInstructions', 'typename', '<new_type_name>') +explain_completion('alterTypeInstructions', 'newcol', '<new_field_name>') + syntax_rules += r''' <alterKeyspaceStatement> ::= "ALTER" ( "KEYSPACE" | "SCHEMA" ) ks=<alterableKeyspaceName> @@ -1170,3 +1235,31 @@ class CqlTableDef: def __str__(self): return '<%s %s.%s>' % (self.__class__.__name__, self.keyspace, self.name) __repr__ = __str__ + +class UserTypesMeta(object): + _meta = {} + + def __init__(self, meta): + self._meta = meta + + @classmethod + def from_layout(cls, layout): + result = {} + for row in layout: + ksname = row[u'keyspace_name'] + if ksname not in result: + result[ksname] = {} + utname = row[u'type_name'] + + result[ksname][utname] = zip(row[u'column_names'], row[u'column_types']) + return cls(meta=result) + + def get_usertypes_names(self, keyspace): + return map(str, self._meta.get(keyspace, {}).keys()) + + def get_field_names(self, keyspace, type): + return [row[0] for row in self._meta.get(keyspace, {}).get(type, [])] + + def get_fields_with_types(self, ksname, typename): + return [(field[0], lookup_casstype(field[1]).cql_parameterized_type()) for field in + self._meta.get(ksname, {}).get(typename, [])] \ No newline at end of file http://git-wip-us.apache.org/repos/asf/cassandra/blob/e5199eea/pylib/cqlshlib/formatting.py ---------------------------------------------------------------------- diff --git a/pylib/cqlshlib/formatting.py b/pylib/cqlshlib/formatting.py index fff75c2..7a0ef17 100644 --- a/pylib/cqlshlib/formatting.py +++ b/pylib/cqlshlib/formatting.py @@ -21,7 +21,6 @@ import math from collections import defaultdict from . import wcwidth from .displaying import colorme, FormattedValue, DEFAULT_VALUE_COLORS -from cql import cqltypes unicode_controlchars_re = re.compile(r'[\x00-\x31\x7f-\xa0]') controlchars_re = re.compile(r'[\x00-\x31\x7f-\xff]') @@ -238,3 +237,22 @@ def format_value_map(val, encoding, colormap, time_format, float_precision, subt + rb displaywidth = 4 * len(subs) + sum(k.displaywidth + v.displaywidth for (k, v) in subs) return FormattedValue(bval, coloredval, displaywidth) + +def format_value_utype(val, encoding, colormap, time_format, float_precision, subtypes, nullval, **_): + def format_field_value(v, subtype): + return format_value(subtype, v, encoding=encoding, colormap=colormap, + time_format=time_format, float_precision=float_precision, + nullval=nullval, quote=True) + + def format_field_name(name): + return format_value_text(name, encoding=encoding, colormap=colormap, quote=False) + + subs = [(format_field_name(k), format_field_value(v, subtypes[index])) for (index, (k, v)) in enumerate(val)] + bval = '{' + ', '.join(k.strval + ': ' + v.strval for (k, v) in subs) + '}' + lb, comma, colon, rb = [colormap['collection'] + s + colormap['reset'] + for s in ('{', ', ', ': ', '}')] + coloredval = lb \ + + comma.join(k.coloredval + colon + v.coloredval for (k, v) in subs) \ + + rb + displaywidth = 4 * len(subs) + sum(k.displaywidth + v.displaywidth for (k, v) in subs) + return FormattedValue(bval, coloredval, displaywidth) \ No newline at end of file http://git-wip-us.apache.org/repos/asf/cassandra/blob/e5199eea/pylib/cqlshlib/usertypes.py ---------------------------------------------------------------------- diff --git a/pylib/cqlshlib/usertypes.py b/pylib/cqlshlib/usertypes.py new file mode 100644 index 0000000..078016b --- /dev/null +++ b/pylib/cqlshlib/usertypes.py @@ -0,0 +1,63 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from cql.marshal import uint16_unpack +from cql.cqltypes import CompositeType +from formatting import formatter_for, format_value_utype + + +def get_field_names(ks_name, ut_name): + """ + UserTypes will use this function to get its fields names from Shell's utypes_dict + """ + raise NotImplementedError("this function shall be overloaded by Shell") + + +class UserType(CompositeType): + typename = 'UserType' + + @classmethod + def apply_parameters(cls, *subtypes): + ksname = subtypes[0].cassname + newname = subtypes[1].cassname.decode("hex") + field_names = get_field_names(ksname, newname) + assert len(field_names) == len(subtypes[2:]) + formatter_for(newname)(format_value_utype) + return type(newname, (cls,), {'subtypes': subtypes[2:], + 'cassname': cls.cassname, 'typename': newname, 'fieldnames': field_names}) + + @classmethod + def cql_parameterized_type(cls): + return cls.typename + + @classmethod + def deserialize_safe(cls, byts): + p = 0 + result = [] + for col_name, col_type in zip(cls.fieldnames, cls.subtypes): + if p == len(byts): + break + itemlen = uint16_unpack(byts[p:p + 2]) + p += 2 + item = byts[p:p + itemlen] + p += itemlen + result.append((str(col_name), col_type.from_binary(item))) + p += 1 + + return result + + +
