Properly decode UDTs with nulls in cqlsh patch by Mikhail Stepura; reviewed by Aleksey Yeschenko for CASSANDRA-7289
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/39c295d8 Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/39c295d8 Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/39c295d8 Branch: refs/heads/trunk Commit: 39c295d803d3af2dddf4c2e98b6a5ea0c523b17e Parents: a277eab Author: Mikhail Stepura <mish...@apache.org> Authored: Fri May 23 20:44:16 2014 -0700 Committer: Mikhail Stepura <mish...@apache.org> Committed: Mon May 26 09:41:55 2014 -0700 ---------------------------------------------------------------------- pylib/cqlshlib/formatting.py | 4 +- pylib/cqlshlib/test/cassconnect.py | 10 ++-- pylib/cqlshlib/test/test_cqlsh_completion.py | 2 +- pylib/cqlshlib/test/test_cqlsh_output.py | 58 ++++++++++++----------- pylib/cqlshlib/test/test_keyspace_init.cql | 10 +++- pylib/cqlshlib/usertypes.py | 9 ++-- 6 files changed, 55 insertions(+), 38 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cassandra/blob/39c295d8/pylib/cqlshlib/formatting.py ---------------------------------------------------------------------- diff --git a/pylib/cqlshlib/formatting.py b/pylib/cqlshlib/formatting.py index 73a6213..1a504ff 100644 --- a/pylib/cqlshlib/formatting.py +++ b/pylib/cqlshlib/formatting.py @@ -246,6 +246,8 @@ formatter_for('OrderedDict')(format_value_map) def format_value_utype(val, encoding, colormap, time_format, float_precision, nullval, **_): def format_field_value(v): + if v is None: + return colorme(nullval, colormap, 'error') return format_value(type(v), v, encoding=encoding, colormap=colormap, time_format=time_format, float_precision=float_precision, nullval=nullval, quote=True) @@ -253,7 +255,7 @@ def format_value_utype(val, encoding, colormap, time_format, float_precision, nu def format_field_name(name): return format_value_text(name, encoding=encoding, colormap=colormap, quote=False) - subs = [(format_field_name(k), format_field_value(v)) for (k, v) in val._asdict().items() if v is not None] + subs = [(format_field_name(k), format_field_value(v)) for (k, v) in val._asdict().items()] bval = '{' + ', '.join(k.strval + ': ' + v.strval for (k, v) in subs) + '}' lb, comma, colon, rb = [colormap['collection'] + s + colormap['reset'] for s in ('{', ', ', ': ', '}')] http://git-wip-us.apache.org/repos/asf/cassandra/blob/39c295d8/pylib/cqlshlib/test/cassconnect.py ---------------------------------------------------------------------- diff --git a/pylib/cqlshlib/test/cassconnect.py b/pylib/cqlshlib/test/cassconnect.py index 6ef6eb9..21dddcd 100644 --- a/pylib/cqlshlib/test/cassconnect.py +++ b/pylib/cqlshlib/test/cassconnect.py @@ -24,15 +24,15 @@ from .run_cqlsh import run_cqlsh, call_cqlsh test_keyspace_init = os.path.join(rundir, 'test_keyspace_init.cql') -def get_cassandra_connection(cql_version=None): +def get_cassandra_connection(cql_version=cqlsh.DEFAULT_CQLVER): if cql_version is None: - cql_version = '3.1.6' + cql_version = cqlsh.DEFAULT_CQLVER conn = cql((TEST_HOST,), TEST_PORT, cql_version=cql_version) # until the cql lib does this for us conn.cql_version = cql_version return conn -def get_cassandra_cursor(cql_version=None): +def get_cassandra_cursor(cql_version=cqlsh.DEFAULT_CQLVER): return get_cassandra_connection(cql_version=cql_version).cursor() TEST_KEYSPACES_CREATED = [] @@ -73,7 +73,7 @@ def execute_cql_file(cursor, fname): return execute_cql_commands(cursor, f.read()) def create_test_db(): - with cassandra_cursor(ks=None, cql_version='3.1.6') as c: + with cassandra_cursor(ks=None) as c: k = create_test_keyspace(c) execute_cql_file(c, test_keyspace_init) return k @@ -83,7 +83,7 @@ def remove_test_db(): c.execute('DROP KEYSPACE %s' % quote_name(TEST_KEYSPACES_CREATED.pop(-1))) @contextlib.contextmanager -def cassandra_connection(cql_version=None): +def cassandra_connection(cql_version=cqlsh.DEFAULT_CQLVER): """ Make a Cassandra CQL connection with the given CQL version and get a cursor for it, and optionally connect to a given keyspace. http://git-wip-us.apache.org/repos/asf/cassandra/blob/39c295d8/pylib/cqlshlib/test/test_cqlsh_completion.py ---------------------------------------------------------------------- diff --git a/pylib/cqlshlib/test/test_cqlsh_completion.py b/pylib/cqlshlib/test/test_cqlsh_completion.py index 221c6b4..2da18d7 100644 --- a/pylib/cqlshlib/test/test_cqlsh_completion.py +++ b/pylib/cqlshlib/test/test_cqlsh_completion.py @@ -37,7 +37,7 @@ completion_separation_re = re.compile(r'\s+') class CqlshCompletionCase(BaseTestCase): def setUp(self): - self.cqlsh_runner = testrun_cqlsh(cqlver=self.cqlver, env={'COLUMNS': '100000'}) + self.cqlsh_runner = testrun_cqlsh(cqlver=cqlsh.DEFAULT_CQLVER, env={'COLUMNS': '100000'}) self.cqlsh = self.cqlsh_runner.__enter__() def tearDown(self): http://git-wip-us.apache.org/repos/asf/cassandra/blob/39c295d8/pylib/cqlshlib/test/test_cqlsh_output.py ---------------------------------------------------------------------- diff --git a/pylib/cqlshlib/test/test_cqlsh_output.py b/pylib/cqlshlib/test/test_cqlsh_output.py index 32cdbe1..d91b4f3 100644 --- a/pylib/cqlshlib/test/test_cqlsh_output.py +++ b/pylib/cqlshlib/test/test_cqlsh_output.py @@ -21,7 +21,7 @@ from __future__ import with_statement import re from itertools import izip -from .basecase import (BaseTestCase, cqlshlog, dedent, at_a_time, cql, +from .basecase import (BaseTestCase, cqlshlog, dedent, at_a_time, cql, cqlsh, TEST_HOST, TEST_PORT) from .cassconnect import (get_test_keyspace, testrun_cqlsh, testcall_cqlsh, cassandra_cursor, split_cql_commands, quote_name) @@ -65,7 +65,7 @@ class TestCqlshOutput(BaseTestCase): % (tags, coloredtext.colored_version(), coloredtext.colortags())) def assertCqlverQueriesGiveColoredOutput(self, queries_and_expected_outputs, - cqlver=(), **kwargs): + cqlver=(cqlsh.DEFAULT_CQLVER,), **kwargs): if not isinstance(cqlver, (tuple, list)): cqlver = (cqlver,) for ver in cqlver: @@ -200,7 +200,7 @@ class TestCqlshOutput(BaseTestCase): (1 rows) nnnnnnnn """), - ), cqlver="3.1.6") + ), cqlver=cqlsh.DEFAULT_CQLVER) q = 'select COUNT(*) FROM twenty_rows_composite_table limit 1000000;' self.assertQueriesGiveColoredOutput(( @@ -216,7 +216,7 @@ class TestCqlshOutput(BaseTestCase): (1 rows) nnnnnnnn """), - ), cqlver="3.1.6") + ), cqlver=cqlsh.DEFAULT_CQLVER) def test_static_cf_output(self): self.assertCqlverQueriesGiveColoredOutput(( @@ -236,7 +236,7 @@ class TestCqlshOutput(BaseTestCase): (3 rows) nnnnnnnn """), - ), cqlver="3.1.6") + ), cqlver=cqlsh.DEFAULT_CQLVER) self.assertQueriesGiveColoredOutput(( ('select * from dynamic_columns;', """ @@ -259,14 +259,14 @@ class TestCqlshOutput(BaseTestCase): (5 rows) nnnnnnnn """), - ), cqlver="3.1.6") + ), cqlver=cqlsh.DEFAULT_CQLVER) def test_empty_cf_output(self): self.assertCqlverQueriesGiveColoredOutput(( ('select * from empty_table;', """ (0 rows) """), - ), cqlver="3.1.6") + ), cqlver=cqlsh.DEFAULT_CQLVER) q = 'select * from has_all_types where num = 999;' @@ -275,7 +275,7 @@ class TestCqlshOutput(BaseTestCase): (q, """ (0 rows) """), - ), cqlver="3.1.6") + ), cqlver=cqlsh.DEFAULT_CQLVER) def test_columnless_key_output(self): q = "select a from twenty_rows_table where a in ('1', '2', '-9192');" @@ -295,7 +295,7 @@ class TestCqlshOutput(BaseTestCase): (2 rows) nnnnnnnn """), - ), cqlver="3.1.6") + ), cqlver=cqlsh.DEFAULT_CQLVER) def test_numeric_output(self): self.assertCqlverQueriesGiveColoredOutput(( @@ -344,7 +344,7 @@ class TestCqlshOutput(BaseTestCase): (5 rows) nnnnnnnn """), - ), cqlver="3.1.6") + ), cqlver=cqlsh.DEFAULT_CQLVER) def test_timestamp_output(self): self.assertQueriesGiveColoredOutput(( @@ -397,7 +397,7 @@ class TestCqlshOutput(BaseTestCase): (4 rows) nnnnnnnn """), - ), cqlver="3.1.6") + ), cqlver=cqlsh.DEFAULT_CQLVER) def test_null_output(self): # column with metainfo but no values @@ -416,7 +416,7 @@ class TestCqlshOutput(BaseTestCase): (2 rows) nnnnnnnn """), - ), cqlver="3.1.6") + ), cqlver=cqlsh.DEFAULT_CQLVER) # all-columns, including a metainfo column has no values (cql3) self.assertQueriesGiveColoredOutput(( @@ -434,7 +434,7 @@ class TestCqlshOutput(BaseTestCase): (2 rows) nnnnnnnn """), - ), cqlver="3.1.6") + ), cqlver=cqlsh.DEFAULT_CQLVER) def test_string_output_ascii(self): self.assertCqlverQueriesGiveColoredOutput(( @@ -458,7 +458,7 @@ class TestCqlshOutput(BaseTestCase): (5 rows) nnnnnnnn """), - ), cqlver="3.1.6") + ), cqlver=cqlsh.DEFAULT_CQLVER) def test_string_output_utf8(self): # many of these won't line up visually here, to keep the source code @@ -492,7 +492,7 @@ class TestCqlshOutput(BaseTestCase): (7 rows) nnnnnnnn """.encode('utf-8')), - ), cqlver="3.1.6", env={'LANG': 'en_US.UTF-8'}) + ), cqlver=cqlsh.DEFAULT_CQLVER, env={'LANG': 'en_US.UTF-8'}) def test_blob_output(self): self.assertCqlverQueriesGiveColoredOutput(( @@ -514,7 +514,7 @@ class TestCqlshOutput(BaseTestCase): (4 rows) nnnnnnnn """), - ), cqlver="3.1.6") + ), cqlver=cqlsh.DEFAULT_CQLVER) def test_colname_decoding_errors(self): # not clear how to achieve this situation in the first place. the @@ -543,7 +543,7 @@ class TestCqlshOutput(BaseTestCase): Failed to decode value '\x00\xff\x00\xff' (for column 'utf8col') as text: 'utf8' codec can't decode byte 0xff in position 1: invalid start byte RRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRR """), - ), cqlver="3.1.6") + ), cqlver=cqlsh.DEFAULT_CQLVER) def test_key_decoding_errors(self): self.assertCqlverQueriesGiveColoredOutput(( @@ -563,10 +563,10 @@ class TestCqlshOutput(BaseTestCase): Failed to decode value '\x00\xff\x02\x8f' (for column 'pkey') as text: 'utf8' codec can't decode byte 0xff in position 1: invalid start byte RRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRR """), - ), cqlver="3.1.6") + ), cqlver=cqlsh.DEFAULT_CQLVER) def test_prompt(self): - with testrun_cqlsh(tty=True, keyspace=None, cqlver="3.1.6") as c: + with testrun_cqlsh(tty=True, keyspace=None, cqlver=cqlsh.DEFAULT_CQLVER) as c: self.assertEqual(c.output_header.splitlines()[-1], 'cqlsh> ') c.send('\n') @@ -594,7 +594,7 @@ class TestCqlshOutput(BaseTestCase): "RRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRR") def test_describe_keyspace_output(self): - fullcqlver = '3.1.6' + fullcqlver = cqlsh.DEFAULT_CQLVER with testrun_cqlsh(tty=True, cqlver=fullcqlver) as c: ks = get_test_keyspace() qks = quote_name(ks) @@ -666,7 +666,7 @@ class TestCqlshOutput(BaseTestCase): """ % quote_name(get_test_keyspace())) - with testrun_cqlsh(tty=True, cqlver='3.1.6') as c: + with testrun_cqlsh(tty=True, cqlver=cqlsh.DEFAULT_CQLVER) as c: for cmdword in ('describe table', 'desc columnfamily'): for semicolon in (';', ''): output = c.cmd_and_response('%s has_all_types%s' % (cmdword, semicolon)) @@ -684,7 +684,7 @@ class TestCqlshOutput(BaseTestCase): ks = get_test_keyspace() - with testrun_cqlsh(tty=True, keyspace=None, cqlver="3.1.6") as c: + with testrun_cqlsh(tty=True, keyspace=None, cqlver=cqlsh.DEFAULT_CQLVER) as c: # when not in a keyspace for cmdword in ('DESCRIBE COLUMNFAMILIES', 'desc tables'): @@ -736,7 +736,7 @@ class TestCqlshOutput(BaseTestCase): \n ''' - with testrun_cqlsh(tty=True, keyspace=None, cqlver="3.1.6") as c: + with testrun_cqlsh(tty=True, keyspace=None, cqlver=cqlsh.DEFAULT_CQLVER) as c: # not in a keyspace for semicolon in ('', ';'): @@ -829,25 +829,29 @@ class TestCqlshOutput(BaseTestCase): MMMMMMMMM -------------------------------------------------------------------------------------------------------------------------------------------- + {{city: 'Chelyabinsk', address: '3rd street', zip: null}, {city: 'Chigirinsk', address: null, zip: '676722'}} + BBYYYYBBYYYYYYYYYYYYYBBYYYYYYYBBYYYYYYYYYYYYBBYYYBBRRRRBBBBYYYYBBYYYYYYYYYYYYBBYYYYYYYBBRRRRBBYYYBBYYYYYYYYBB {{city: 'Austin', address: '902 East 5th St. #202', zip: '78702'}, {city: 'Sunnyvale', address: '292 Gibraltar Drive #107', zip: '94089'}} BBYYYYBBYYYYYYYYBBYYYYYYYBBYYYYYYYYYYYYYYYYYYYYYYYBBYYYBBYYYYYYYBBBBYYYYBBYYYYYYYYYYYBBYYYYYYYBBYYYYYYYYYYYYYYYYYYYYYYYYYYBBYYYBBYYYYYYYBB - (1 rows) + (2 rows) nnnnnnnn """), - ), cqlver="3.1.6") + ), cqlver=cqlsh.DEFAULT_CQLVER) self.assertCqlverQueriesGiveColoredOutput(( ("select phone_numbers from users;", r""" phone_numbers MMMMMMMMMMMMM ------------------------------------------------------------------------------------- + {{country: null, number: '03'}, {country: '+7', number: null}} + BBYYYYYYYBBRRRRBBYYYYYYBBYYYYBBBBYYYYYYYBBYYYYBBYYYYYYBBRRRRBB {{country: '+1', number: '512-537-7809'}, {country: '+44', number: '208 622 3021'}} BBYYYYYYYBBYYYYBBYYYYYYBBYYYYYYYYYYYYYYBBBBYYYYYYYBBYYYYYBBYYYYYYBBYYYYYYYYYYYYYYBB - (1 rows) + (2 rows) nnnnnnnn """), - ), cqlver="3.1.6") \ No newline at end of file + ), cqlver=cqlsh.DEFAULT_CQLVER) \ No newline at end of file http://git-wip-us.apache.org/repos/asf/cassandra/blob/39c295d8/pylib/cqlshlib/test/test_keyspace_init.cql ---------------------------------------------------------------------- diff --git a/pylib/cqlshlib/test/test_keyspace_init.cql b/pylib/cqlshlib/test/test_keyspace_init.cql index 6f44e92..2c3d81d 100644 --- a/pylib/cqlshlib/test/test_keyspace_init.cql +++ b/pylib/cqlshlib/test/test_keyspace_init.cql @@ -222,4 +222,12 @@ values ('jbellis', {{city: 'Austin', address: '902 East 5th St. #202', zip: '78702'}, {city: 'Sunnyvale', address: '292 Gibraltar Drive #107', zip: '94089'}}, {{country: '+44', number: '208 622 3021'}, - {country: '+1', number: '512-537-7809'}}); \ No newline at end of file + {country: '+1', number: '512-537-7809'}}); + +insert into users (login, name, addresses, phone_numbers) +values ('vpupkin', + 'vasya pupkin', + {{city: 'Chelyabinsk', address: '3rd street', zip: null}, + {city: 'Chigirinsk', address: null, zip: '676722'}}, + {{country: '+7', number: null}, + {country: null, number: '03'}}); http://git-wip-us.apache.org/repos/asf/cassandra/blob/39c295d8/pylib/cqlshlib/usertypes.py ---------------------------------------------------------------------- diff --git a/pylib/cqlshlib/usertypes.py b/pylib/cqlshlib/usertypes.py index 22d3080..577a465 100644 --- a/pylib/cqlshlib/usertypes.py +++ b/pylib/cqlshlib/usertypes.py @@ -47,9 +47,12 @@ class UserType(CompositeType): break itemlen = int32_unpack(byts[p:p + cls.FIELD_LENGTH]) p += cls.FIELD_LENGTH - item = byts[p:p + itemlen] - p += itemlen - result.append(col_type.from_binary(item)) + if itemlen < 0: + result.append(None) + else: + item = byts[p:p + itemlen] + p += itemlen + result.append(col_type.from_binary(item)) if len(result) < len(cls.subtypes): nones = [None] * (len(cls.subtypes) - len(result))