Merge branch 'cassandra-3.0' into trunk
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/e6505a60 Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/e6505a60 Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/e6505a60 Branch: refs/heads/trunk Commit: e6505a606f98707d3aab436e0713f5dab004a5ab Parents: 0b81f72 f2afd04 Author: Stefania Alborghetti <[email protected]> Authored: Wed Apr 27 09:09:34 2016 +0800 Committer: Stefania Alborghetti <[email protected]> Committed: Wed Apr 27 09:10:17 2016 +0800 ---------------------------------------------------------------------- CHANGES.txt | 1 + pylib/cqlshlib/copyutil.py | 39 +++++++++++++++++++++++++++++---------- pylib/cqlshlib/formatting.py | 22 +++++++++++++++++++++- 3 files changed, 51 insertions(+), 11 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cassandra/blob/e6505a60/CHANGES.txt ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cassandra/blob/e6505a60/pylib/cqlshlib/copyutil.py ---------------------------------------------------------------------- diff --cc pylib/cqlshlib/copyutil.py index 36c951c,b17bdfe..46c7eb6 --- a/pylib/cqlshlib/copyutil.py +++ b/pylib/cqlshlib/copyutil.py @@@ -306,9 -306,8 +306,10 @@@ class CopyTask(object) copy_options['pagetimeout'] = int(opts.pop('pagetimeout', max(10, 10 * (copy_options['pagesize'] / 1000)))) copy_options['maxattempts'] = int(opts.pop('maxattempts', 5)) copy_options['dtformats'] = DateTimeFormat(opts.pop('datetimeformat', shell.display_timestamp_format), -- shell.display_date_format, shell.display_nanotime_format) - copy_options['float_precision'] = shell.display_float_precision ++ shell.display_date_format, shell.display_nanotime_format, ++ milliseconds_only=True) + copy_options['floatprecision'] = int(opts.pop('floatprecision', '5')) + copy_options['doubleprecision'] = int(opts.pop('doubleprecision', '12')) copy_options['chunksize'] = int(opts.pop('chunksize', 5000)) copy_options['ingestrate'] = int(opts.pop('ingestrate', 100000)) copy_options['maxbatchsize'] = int(opts.pop('maxbatchsize', 20)) http://git-wip-us.apache.org/repos/asf/cassandra/blob/e6505a60/pylib/cqlshlib/formatting.py ---------------------------------------------------------------------- diff --cc pylib/cqlshlib/formatting.py index aa50204,dcd08da..5364c18 --- a/pylib/cqlshlib/formatting.py +++ b/pylib/cqlshlib/formatting.py @@@ -101,101 -99,20 +101,102 @@@ def color_text(bval, colormap, displayw DEFAULT_NANOTIME_FORMAT = '%H:%M:%S.%N' DEFAULT_DATE_FORMAT = '%Y-%m-%d' -DEFAULT_TIMESTAMP_FORMAT = '%Y-%m-%d %H:%M:%S%z' -if platform.system() == 'Windows': - DEFAULT_TIME_FORMAT = '%Y-%m-%d %H:%M:%S %Z' +DEFAULT_TIMESTAMP_FORMAT = os.environ.get('CQLSH_DEFAULT_TIMESTAMP_FORMAT', '') +if not DEFAULT_TIMESTAMP_FORMAT: + DEFAULT_TIMESTAMP_FORMAT = '%Y-%m-%d %H:%M:%S.%f%z' -class DateTimeFormat(): +class DateTimeFormat: def __init__(self, timestamp_format=DEFAULT_TIMESTAMP_FORMAT, date_format=DEFAULT_DATE_FORMAT, -- nanotime_format=DEFAULT_NANOTIME_FORMAT, timezone=None): ++ nanotime_format=DEFAULT_NANOTIME_FORMAT, timezone=None, milliseconds_only=False): self.timestamp_format = timestamp_format self.date_format = date_format self.nanotime_format = nanotime_format self.timezone = timezone ++ self.milliseconds_only = milliseconds_only # the microseconds part, .NNNNNN, wil be rounded to .NNN + + +class CqlType(object): + """ + A class for converting a string into a cql type name that can match a formatter + and a list of its sub-types, if any. + """ + pattern = re.compile('^([^<]*)<(.*)>$') # *<*> + + def __init__(self, typestring, ksmeta=None): + self.type_name, self.sub_types, self.formatter = self.parse(typestring, ksmeta) + + def __str__(self): + return "%s%s" % (self.type_name, self.sub_types or '') + + __repr__ = __str__ + + def get_n_sub_types(self, num): + """ + Return the sub-types if the requested number matches the length of the sub-types (tuples) + or the first sub-type times the number requested if the length of the sub-types is one (list, set), + otherwise raise an exception + """ + if len(self.sub_types) == num: + return self.sub_types + elif len(self.sub_types) == 1: + return [self.sub_types[0]] * num + else: + raise Exception("Unexpected number of subtypes %d - %s" % (num, self.sub_types)) + + def parse(self, typestring, ksmeta): + """ + Parse the typestring by looking at this pattern: *<*>. If there is no match then the type + is either a simple type or a user type, otherwise it must be a composite type + for which we need to look-up the sub-types. For user types the sub types can be extracted + from the keyspace metadata. + """ + while True: + m = self.pattern.match(typestring) + if not m: # no match, either a simple or a user type + name = typestring + if ksmeta and name in ksmeta.user_types: # a user type, look at ks meta for sub types + sub_types = [CqlType(t, ksmeta) for t in ksmeta.user_types[name].field_types] + return name, sub_types, format_value_utype + else: + return name, [], self._get_formatter(name) + else: + if m.group(1) == 'frozen': # ignore frozen<> + typestring = m.group(2) + continue + + name = m.group(1) # a composite type, parse sub types + return name, self.parse_sub_types(m.group(2), ksmeta), self._get_formatter(name) + + @staticmethod + def _get_formatter(name): + return _formatters.get(name.lower()) + + @staticmethod + def parse_sub_types(val, ksmeta): + """ + Split val into sub-strings separated by commas but only if not within a <> pair + Return a list of CqlType instances where each instance is initialized with the sub-strings + that were found. + """ + last = 0 + level = 0 + ret = [] + for i, c in enumerate(val): + if c == '<': + level += 1 + elif c == '>': + level -= 1 + elif c == ',' and level == 0: + ret.append(val[last:i].strip()) + last = i + 1 + + if last < len(val) - 1: + ret.append(val[last:].strip()) + + return [CqlType(r, ksmeta) for r in ret] def format_value_default(val, colormap, **_): @@@ -331,14 -236,7 +332,16 @@@ formatter_for('varint')(format_integer_ @formatter_for('datetime') def format_value_timestamp(val, colormap, date_time_format, quote=False, **_): - bval = strftime(date_time_format.timestamp_format, calendar.timegm(val.utctimetuple()), timezone=date_time_format.timezone) + if isinstance(val, datetime.datetime): + bval = strftime(date_time_format.timestamp_format, + calendar.timegm(val.utctimetuple()), + microseconds=val.microsecond, + timezone=date_time_format.timezone) ++ if date_time_format.milliseconds_only: ++ bval = round_microseconds(bval) + else: + bval = str(val) + if quote: bval = "'%s'" % bval return colorme(bval, colormap, 'timestamp') @@@ -353,6 -248,6 +356,23 @@@ def strftime(time_format, seconds, micr ret_dt = ret_dt.astimezone(timezone) return ret_dt.strftime(time_format) ++microseconds_regex = re.compile("(.*)(?:\.(\d{1,6}))(.*)") ++ ++ ++def round_microseconds(val): ++ """ ++ For COPY TO, we need to round microsecond to milliseconds because server side ++ TimestampSerializer.dateStringPatterns only parses milliseconds. If we keep microseconds, ++ users may try to import with COPY FROM a file generated with COPY TO and have problems if ++ prepared statements are disabled, see CASSANDRA-11631. ++ """ ++ m = microseconds_regex.match(val) ++ if not m: ++ return val ++ ++ milliseconds = int(m.group(2)) * pow(10, 3 - len(m.group(2))) ++ return '%s.%03d%s' % (m.group(1), milliseconds, '' if not m.group(3) else m.group(3)) ++ @formatter_for('Date') def format_value_date(val, colormap, **_):
