token-generator - generated tokens too long Patch by Robert Stupp; Reviewed by Stefania for CASSANDRA-9300
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/1611ef3d Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/1611ef3d Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/1611ef3d Branch: refs/heads/cassandra-2.1 Commit: 1611ef3d1995337998b9d7514c7ea5c35fa59895 Parents: 418deaf Author: Robert Stupp <sn...@snazy.de> Authored: Thu May 14 10:35:35 2015 +0200 Committer: Robert Stupp <sn...@snazy.de> Committed: Thu May 14 10:35:35 2015 +0200 ---------------------------------------------------------------------- CHANGES.txt | 3 +++ tools/bin/token-generator | 53 +++++++++++++++++++++++++++----------- tools/bin/token-generator.bat | 34 ++++++++++++++++++++++++ 3 files changed, 75 insertions(+), 15 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cassandra/blob/1611ef3d/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index d7d01cf..cee28bc 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,3 +1,6 @@ +2.0.16: + * token-generator - generated tokens too long (CASSANDRA-9300) + 2.0.15: * Fix counting of tombstones for TombstoneOverwhelmingException (CASSANDRA-9299) * Fix ReconnectableSnitch reconnecting to peers during upgrade (CASSANDRA-6702) http://git-wip-us.apache.org/repos/asf/cassandra/blob/1611ef3d/tools/bin/token-generator ---------------------------------------------------------------------- diff --git a/tools/bin/token-generator b/tools/bin/token-generator index c958529..b70866d 100755 --- a/tools/bin/token-generator +++ b/tools/bin/token-generator @@ -30,24 +30,38 @@ from tempfile import NamedTemporaryFile description = '''Given a list of numbers indicating the number of nodes in each separate datacenter, outputs a recommended list of tokens to use -with RandomPartitioner: one for each node in each datacenter. +with Murmur3Partitioner (by default) or with RandomPartitioner. +The list contains one token for each node in each datacenter. ''' usage = "%prog <nodes_in_dc1> [<nodes_in_dc2> [...]]" parser = optparse.OptionParser(description=description, usage=usage) + +def part_murmur3(option, opt, value, parser): + parser.values.ringoffset=-(1<<63) + parser.values.ringrange=(1<<64) + return +def part_random(option, opt, value, parser): + parser.values.ringoffset=0 + parser.values.ringrange=(1<<127) + return +parser.add_option('--murmur3', action='callback', callback=part_murmur3, + help='Generate tokens for Murmur3Partitioner (default).') +parser.add_option('--random', action='callback', callback=part_random, + help='Generate tokens for RandomPartitioner.') +parser.add_option('--ringoffset', type='int', + help=optparse.SUPPRESS_HELP) parser.add_option('--ringrange', type='int', - help='Specify a numeric maximum token value for your ring, ' - 'different from the default value of 2^127.') + help=optparse.SUPPRESS_HELP) + parser.add_option('--graph', action='store_true', help='Show a rendering of the generated tokens as line ' 'segments in a circle, colored according to datacenter') parser.add_option('-n', '--nts', action='store_const', dest='strat', const='nts', - help='Optimize multi-cluster distribution for ' - 'NetworkTopologyStrategy [default]') + help=optparse.SUPPRESS_HELP) parser.add_option('-o', '--onts', action='store_const', dest='strat', const='onts', - help='Optimize multi-cluster distribution for ' - 'OldNetworkTopologyStrategy') + help=optparse.SUPPRESS_HELP) parser.add_option('--test', action='store_true', help='Run in test mode, outputting an HTML file to display ' @@ -58,8 +72,11 @@ parser.add_option('--browser-wait-time', type='float', help=optparse.SUPPRESS_HE parser.add_option('--test-colors', help=optparse.SUPPRESS_HELP) parser.add_option('--test-graphsize', type='int', help=optparse.SUPPRESS_HELP) + parser.set_defaults( - ringrange=(1<<127), + # default is Murmur3 + ringoffset=-(1<<63), + ringrange=(1<<64), # whether to create (and try to display) graph output graph=False, @@ -94,7 +111,8 @@ class Ring: MIN_DC_OFFSET_DIVIDER = 235 offset_spacer = 2 - def __init__(self, dc_counts, ringrange, strategy='nts'): + def __init__(self, dc_counts, ringoffset, ringrange, strategy='nts'): + self.ringoffset = ringoffset self.ringrange = ringrange self.dc_counts = dc_counts self.calculate_offset_tokens = getattr(self, 'calc_offset_tokens_' + strategy) @@ -110,13 +128,18 @@ class Ring: division = max(lowest_division, self.MIN_DC_OFFSET_DIVIDER) return -self.ringrange // division + def bound_token(self, tok): + if tok < self.ringoffset: + tok += self.ringrange + return tok + def calc_offset_tokens_nts(self): dc_offset = self.best_per_dc_offset() dcs = [] for (dcnum, dccount) in enumerate(self.dc_counts): offset = dcnum * dc_offset arcsize = self.ringrange // (dccount or 1) - dcs.append([(n * arcsize + offset) % self.ringrange for n in xrange(dccount)]) + dcs.append(sorted([self.bound_token((n * arcsize + offset) - self.ringoffset % self.ringrange) for n in xrange(dccount)])) return dcs def calc_offset_tokens_onts(self): @@ -127,7 +150,7 @@ class Ring: final = [[] for x in dcs_by_count] for pos, dc in enumerate(layout): - final[dc].append(pos * self.ringrange // len(layout)) + final[dc].append(self.ringoffset + pos * self.ringrange // len(layout)) return final @@ -139,8 +162,8 @@ def print_tokens(tokens, tokenwidth, indent=0): for tnum, tok in enumerate(toklist): print "%s Node #%0*d: % *d" % (indentstr, nwidth, tnum + 1, tokenwidth, tok) -def calculate_ideal_tokens(datacenters, ringrange, strategy): - return Ring(datacenters, ringrange, strategy).calculate_offset_tokens() +def calculate_ideal_tokens(datacenters, ringoffset, ringrange, strategy): + return Ring(datacenters, ringoffset, ringrange, strategy).calculate_offset_tokens() def file_to_url(path): path = os.path.abspath(path) @@ -248,7 +271,7 @@ def run_tests(opts): tokensets = [] for test in tests: print "Test %r" % (test,) - tokens = calculate_ideal_tokens(test, opts.ringrange, opts.strat) + tokens = calculate_ideal_tokens(test, opts.ringoffset, opts.ringrange, opts.strat) print_tokens(tokens, len(str(opts.ringrange)) + 1, indent=2) tokensets.append(tokens) return tokensets @@ -315,7 +338,7 @@ def main(opts, args): parser.error('Arguments should be integers.') renderer = RingRenderer(ringrange=opts.ringrange, graphsize=opts.graphsize, colors=opts.colorlist) - tokens = calculate_ideal_tokens(datacenters, opts.ringrange, opts.strat) + tokens = calculate_ideal_tokens(datacenters, opts.ringoffset, opts.ringrange, opts.strat) print_tokens(tokens, len(str(opts.ringrange)) + 1) tokensets = [tokens] http://git-wip-us.apache.org/repos/asf/cassandra/blob/1611ef3d/tools/bin/token-generator.bat ---------------------------------------------------------------------- diff --git a/tools/bin/token-generator.bat b/tools/bin/token-generator.bat new file mode 100644 index 0000000..a7188db --- /dev/null +++ b/tools/bin/token-generator.bat @@ -0,0 +1,34 @@ +@ECHO OFF +@REM +@REM Licensed to the Apache Software Foundation (ASF) under one or more +@REM contributor license agreements. See the NOTICE file distributed with +@REM this work for additional information regarding copyright ownership. +@REM The ASF licenses this file to You under the Apache License, Version 2.0 +@REM (the "License"); you may not use this file except in compliance with +@REM the License. You may obtain a copy of the License at +@REM +@REM http://www.apache.org/licenses/LICENSE-2.0 +@REM +@REM Unless required by applicable law or agreed to in writing, software +@REM distributed under the License is distributed on an "AS IS" BASIS, +@REM WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@REM See the License for the specific language governing permissions and +@REM limitations under the License. + +@echo off + +if "%OS%" == "Windows_NT" setlocal + +python -V >nul 2>&1 +if ERRORLEVEL 1 goto err + +python "%~dp0\token-generator" %* +goto finally + +:err +echo Can't detect Python version! + +:finally + +ENDLOCAL +