This is an automated email from the ASF dual-hosted git repository. mck pushed a commit to branch cassandra-3.11 in repository https://gitbox.apache.org/repos/asf/cassandra.git
commit 9e513390b7efd8b6de25be4600499d5a8fb2375d Merge: ec9b7b9 35114d5 Author: Mick Semb Wever <[email protected]> AuthorDate: Sat Mar 27 00:33:18 2021 +0100 Merge branch 'cassandra-3.0' into cassandra-3.11 doc/source/_util/cql.py | 17 +++++++++++++++++ pylib/cassandra-cqlsh-tests.sh | 17 +++++++++++++++++ .../cassandra/stress/report/TimingIntervals.java | 20 ++++++++++++++++++-- 3 files changed, 52 insertions(+), 2 deletions(-) diff --cc doc/source/_util/cql.py index b1c8cde,0000000..673fbcf mode 100644,000000..100644 --- a/doc/source/_util/cql.py +++ b/doc/source/_util/cql.py @@@ -1,267 -1,0 +1,284 @@@ +# -*- coding: utf-8 -*- ++# ++# Licensed to the Apache Software Foundation (ASF) under one ++# or more contributor license agreements. See the NOTICE file ++# distributed with this work for additional information ++# regarding copyright ownership. The ASF licenses this file ++# to you under the Apache License, Version 2.0 (the ++# "License"); you may not use this file except in compliance ++# with the License. You may obtain a copy of the License at ++# ++# http://www.apache.org/licenses/LICENSE-2.0 ++# ++# Unless required by applicable law or agreed to in writing, software ++# distributed under the License is distributed on an "AS IS" BASIS, ++# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++# See the License for the specific language governing permissions and ++# limitations under the License. ++# +""" + CQL pygments lexer + ~~~~~~~~~~~~~~~~~~ + + Lexer for the Cassandra Query Language (CQL). + + This is heavily inspired from the pygments SQL lexer (and the Postgres one in particular) but adapted to CQL + keywords and specificities. + + TODO: This has been hacked quickly, but once it's more tested, we could submit it upstream. + In particular, we have alot of keywords whose meaning depends on the context and we could potentially improve + their handling. For instance, SET is a keyword, but also a type name (that's why currently we also consider + map and list as keywords, not types; we could disambiguate by looking if there is a '<' afterwards). Or things + like USERS, which can is used in some documentation example as a table name but is a keyword too (we could + only consider it a keyword if after LIST for instance). Similarly, type nanes are not reserved, so they and + are sometime used as column identifiers (also, timestamp is both a type and a keyword). I "think" we can + somewhat disambiguate through "states", but unclear how far it's worth going. + + We could also add the predefined functions? +""" + +import re + +from pygments.lexer import Lexer, RegexLexer, do_insertions, bygroups, words +from pygments.token import Punctuation, Whitespace, Error, \ + Text, Comment, Operator, Keyword, Name, String, Number, Generic, Literal +from pygments.lexers import get_lexer_by_name, ClassNotFound +from pygments.util import iteritems + +__all__ = [ 'CQLLexer' ] + +language_re = re.compile(r"\s+LANGUAGE\s+'?(\w+)'?", re.IGNORECASE) + +KEYWORDS = ( + 'SELECT', + 'FROM', + 'AS', + 'WHERE', + 'AND', + 'KEY', + 'KEYS', + 'ENTRIES', + 'FULL', + 'INSERT', + 'UPDATE', + 'WITH', + 'LIMIT', + 'PER', + 'PARTITION', + 'USING', + 'USE', + 'DISTINCT', + 'COUNT', + 'SET', + 'BEGIN', + 'UNLOGGED', + 'BATCH', + 'APPLY', + 'TRUNCATE', + 'DELETE', + 'IN', + 'CREATE', + 'KEYSPACE', + 'SCHEMA', + 'KEYSPACES', + 'COLUMNFAMILY', + 'TABLE', + 'MATERIALIZED', + 'VIEW', + 'INDEX', + 'CUSTOM', + 'ON', + 'TO', + 'DROP', + 'PRIMARY', + 'INTO', + 'VALUES', + 'TIMESTAMP', + 'TTL', + 'CAST', + 'ALTER', + 'RENAME', + 'ADD', + 'TYPE', + 'COMPACT', + 'STORAGE', + 'ORDER', + 'BY', + 'ASC', + 'DESC', + 'ALLOW', + 'FILTERING', + 'IF', + 'IS', + 'CONTAINS', + 'GRANT', + 'ALL', + 'PERMISSION', + 'PERMISSIONS', + 'OF', + 'REVOKE', + 'MODIFY', + 'AUTHORIZE', + 'DESCRIBE', + 'EXECUTE', + 'NORECURSIVE', + 'MBEAN', + 'MBEANS', + 'USER', + 'USERS', + 'ROLE', + 'ROLES', + 'SUPERUSER', + 'NOSUPERUSER', + 'PASSWORD', + 'LOGIN', + 'NOLOGIN', + 'OPTIONS', + 'CLUSTERING', + 'TOKEN', + 'WRITETIME', + 'NULL', + 'NOT', + 'EXISTS', + 'MAP', + 'LIST', + 'NAN', + 'INFINITY', + 'TUPLE', + 'TRIGGER', + 'STATIC', + 'FROZEN', + 'FUNCTION', + 'FUNCTIONS', + 'AGGREGATE', + 'SFUNC', + 'STYPE', + 'FINALFUNC', + 'INITCOND', + 'RETURNS', + 'CALLED', + 'INPUT', + 'LANGUAGE', + 'OR', + 'REPLACE', + 'JSON', + 'LIKE', +) + +DATATYPES = ( + 'ASCII', + 'BIGINT', + 'BLOB', + 'BOOLEAN', + 'COUNTER', + 'DATE', + 'DECIMAL', + 'DOUBLE', + 'EMPTY', + 'FLOAT', + 'INET', + 'INT', + 'SMALLINT', + 'TEXT', + 'TIME', + 'TIMESTAMP', + 'TIMEUUID', + 'TINYINT', + 'UUID', + 'VARCHAR', + 'VARINT', +) + +def language_callback(lexer, match): + """Parse the content of a $-string using a lexer + + The lexer is chosen looking for a nearby LANGUAGE or assumed as + java if no LANGUAGE has been found. + """ + l = None + m = language_re.match(lexer.text[max(0, match.start()-100):match.start()]) + if m is not None: + l = lexer._get_lexer(m.group(1)) + else: + l = lexer._get_lexer('java') + + # 1 = $, 2 = delimiter, 3 = $ + yield (match.start(1), String, match.group(1)) + yield (match.start(2), String.Delimiter, match.group(2)) + yield (match.start(3), String, match.group(3)) + # 4 = string contents + if l: + for x in l.get_tokens_unprocessed(match.group(4)): + yield x + else: + yield (match.start(4), String, match.group(4)) + # 5 = $, 6 = delimiter, 7 = $ + yield (match.start(5), String, match.group(5)) + yield (match.start(6), String.Delimiter, match.group(6)) + yield (match.start(7), String, match.group(7)) + + +class CQLLexer(RegexLexer): + """ + Lexer for the Cassandra Query Language. + """ + + name = 'Cassandra Query Language' + aliases = ['cql'] + filenames = ['*.cql'] + mimetypes = ['text/x-cql'] + + flags = re.IGNORECASE + tokens = { + 'root': [ + (r'\s+', Text), + (r'--.*\n?', Comment.Single), + (r'//.*\n?', Comment.Single), + (r'/\*', Comment.Multiline, 'multiline-comments'), + (r'(' + '|'.join(s.replace(" ", "\s+") + for s in DATATYPES) + + r')\b', Name.Builtin), + (words(KEYWORDS, suffix=r'\b'), Keyword), + (r'[+*/<>=~!@#%^&|`?-]+', Operator), + (r'\$\d+', Name.Variable), + + # Using Number instead of the more accurate Literal because the latter don't seem to e highlighted in most + # styles + (r'[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}', Number), # UUIDs + (r'0x[0-9a-fA-F]+', Number), # Blobs + + (r'([0-9]*\.[0-9]*|[0-9]+)(e[+-]?[0-9]+)?', Number.Float), + (r'[0-9]+', Number.Integer), + (r"((?:E|U&)?)(')", bygroups(String.Affix, String.Single), 'string'), + # quoted identifier + (r'((?:U&)?)(")', bygroups(String.Affix, String.Name), 'quoted-ident'), + (r'(?s)(\$)([^$]*)(\$)(.*?)(\$)(\2)(\$)', language_callback), + (r'[a-z_]\w*', Name), + (r'[;:()\[\]{},.]', Punctuation), + ], + 'multiline-comments': [ + (r'/\*', Comment.Multiline, 'multiline-comments'), + (r'\*/', Comment.Multiline, '#pop'), + (r'[^/*]+', Comment.Multiline), + (r'[/*]', Comment.Multiline) + ], + 'string': [ + (r"[^']+", String.Single), + (r"''", String.Single), + (r"'", String.Single, '#pop'), + ], + 'quoted-ident': [ + (r'[^"]+', String.Name), + (r'""', String.Name), + (r'"', String.Name, '#pop'), + ], + } + + def get_tokens_unprocessed(self, text, *args): + # Have a copy of the entire text to be used by `language_callback`. + self.text = text + for x in RegexLexer.get_tokens_unprocessed(self, text, *args): + yield x + + def _get_lexer(self, lang): + return get_lexer_by_name(lang, **self.options) diff --cc pylib/cassandra-cqlsh-tests.sh index 8174636,a219485..08c711b --- a/pylib/cassandra-cqlsh-tests.sh +++ b/pylib/cassandra-cqlsh-tests.sh @@@ -1,4 -1,20 +1,21 @@@ #!/bin/bash -x + # + # Licensed to the Apache Software Foundation (ASF) under one + # or more contributor license agreements. See the NOTICE file + # distributed with this work for additional information + # regarding copyright ownership. The ASF licenses this file + # to you under the Apache License, Version 2.0 (the + # "License"); you may not use this file except in compliance + # with the License. You may obtain a copy of the License at + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. ++# ################################ # diff --cc tools/stress/src/org/apache/cassandra/stress/report/TimingIntervals.java index 747a42a,0000000..400fab9 mode 100644,000000..100644 --- a/tools/stress/src/org/apache/cassandra/stress/report/TimingIntervals.java +++ b/tools/stress/src/org/apache/cassandra/stress/report/TimingIntervals.java @@@ -1,128 -1,0 +1,144 @@@ ++/* ++ * Licensed to the Apache Software Foundation (ASF) under one ++ * or more contributor license agreements. See the NOTICE file ++ * distributed with this work for additional information ++ * regarding copyright ownership. The ASF licenses this file ++ * to you under the Apache License, Version 2.0 (the ++ * "License"); you may not use this file except in compliance ++ * with the License. You may obtain a copy of the License at ++ * ++ * http://www.apache.org/licenses/LICENSE-2.0 ++ * ++ * Unless required by applicable law or agreed to in writing, software ++ * distributed under the License is distributed on an "AS IS" BASIS, ++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++ * See the License for the specific language governing permissions and ++ * limitations under the License. ++ */ ++ +package org.apache.cassandra.stress.report; + - import java.util.Arrays; +import java.util.Map; - import java.util.TreeMap; + +public class TimingIntervals +{ + final Map<String, TimingInterval> intervals; + + public TimingIntervals(Map<String, TimingInterval> intervals) + { + this.intervals = intervals; + } + + public TimingInterval get(String opType) + { + return intervals.get(opType); + } + + + public String str(TimingInterval.TimingParameter value, String unit) + { + return str(value, Double.NaN, unit); + } + + public String str(TimingInterval.TimingParameter value, double rank, String unit) + { + if (intervals.size() == 0) + { + return "[]"; + } + + StringBuilder sb = new StringBuilder("["); + + for (Map.Entry<String, TimingInterval> entry : intervals.entrySet()) + { + sb.append(entry.getKey()); + sb.append(": "); + sb.append(entry.getValue().getStringValue(value, rank)); + if (unit.length() > 0) + { + sb.append(" "); + sb.append(unit); + } + sb.append(", "); + } + + sb.setLength(sb.length()-2); + sb.append("]"); + + return sb.toString(); + } + + public String opRates() + { + return str(TimingInterval.TimingParameter.OPRATE, "op/s"); + } + + public String partitionRates() + { + return str(TimingInterval.TimingParameter.PARTITIONRATE, "pk/s"); + } + + public String rowRates() + { + return str(TimingInterval.TimingParameter.ROWRATE, "row/s"); + } + + public String meanLatencies() + { + return str(TimingInterval.TimingParameter.MEANLATENCY, "ms"); + } + + public String maxLatencies() + { + return str(TimingInterval.TimingParameter.MAXLATENCY, "ms"); + } + + public String medianLatencies() + { + return str(TimingInterval.TimingParameter.MEDIANLATENCY, "ms"); + } + + public String latenciesAtPercentile(double rank) + { + return str(TimingInterval.TimingParameter.RANKLATENCY, rank, "ms"); + } + + public String errorCounts() + { + return str(TimingInterval.TimingParameter.ERRORCOUNT, ""); + } + + public String partitionCounts() + { + return str(TimingInterval.TimingParameter.PARTITIONCOUNT, ""); + } + + public long opRate() + { + long v = 0; + for (TimingInterval interval : intervals.values()) + v += interval.opRate(); + return v; + } + + public long startNanos() + { + long start = Long.MAX_VALUE; + for (TimingInterval interval : intervals.values()) + start = Math.min(start, interval.startNanos()); + return start; + } + + public long endNanos() + { + long end = Long.MIN_VALUE; + for (TimingInterval interval : intervals.values()) + end = Math.max(end, interval.startNanos()); + return end; + } + + public Map<String, TimingInterval> intervals() + { + return intervals; + } +} --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
