2 new revisions:
Revision: 727d434767d0
Branch: default
Author: Pekka Klärck
Date: Tue Apr 22 14:15:18 2014 UTC
Log: Support giving seed to --randomize....
http://code.google.com/p/robotframework/source/detail?r=727d434767d0
Revision: 14bbd457f8dc
Branch: default
Author: Pekka Klärck
Date: Tue Apr 22 15:56:54 2014 UTC
Log: Clean-up and fix giving seed to --randomize....
http://code.google.com/p/robotframework/source/detail?r=14bbd457f8dc
==============================================================================
Revision: 727d434767d0
Branch: default
Author: Pekka Klärck
Date: Tue Apr 22 14:15:18 2014 UTC
Log: Support giving seed to --randomize.
Applied a patch by Lionel Perrin.
Update issue 1673
Patch applied. Also added Lionel to AUTHORS.txt.
http://code.google.com/p/robotframework/source/detail?r=727d434767d0
Modified:
/AUTHORS.txt
/atest/robot/cli/runner/randomize.txt
/src/robot/conf/settings.py
/src/robot/run.py
/src/robot/running/model.py
/src/robot/running/randomizer.py
/utest/running/test_randomizer.py
=======================================
--- /AUTHORS.txt Tue Dec 3 12:10:54 2013 UTC
+++ /AUTHORS.txt Tue Apr 22 14:15:18 2014 UTC
@@ -43,3 +43,4 @@
*varargs support using java.util.List
(2.8.3)
Mirel Pehadzic Terminal emulation for Telnet library
(2.8.2)
Diogo Sa-Chaves De Oliveira Terminal emulation for Telnet library
(2.8.2)
+Lionel Perrin Giving custom seed to --randomize (2.8.5)
=======================================
--- /atest/robot/cli/runner/randomize.txt Thu May 30 10:59:56 2013 UTC
+++ /atest/robot/cli/runner/randomize.txt Tue Apr 22 14:15:18 2014 UTC
@@ -23,6 +23,18 @@
Should Not Be Equal As Strings ${SUITE.suites} ${DEFAULT SUITE ORDER}
${tests} = Get Tests
Should Not Be Equal As Strings ${tests} ${DEFAULT TEST ORDER}
+
+Randomizing Suites And Tests with seed
+ [Setup] Run Tests --randomize all:1234 misc/multiple_suites
+ ${suites1} = Get Suites
+ ${tests1} = Get Tests
+ Should Not Be Equal As Strings ${suites1} ${DEFAULT SUITE ORDER}
+ Should Not Be Equal As Strings ${tests1} ${DEFAULT TEST ORDER}
+ Run Tests --randomize all:1234 misc/multiple_suites
+ ${suites2} = Get Suites
+ ${tests2} = Get Tests
+ Should Be Equal As Strings ${suites1} ${suites2}
+ Should Be Equal As Strings ${tests1} ${tests2}
Last Option Overrides All Previous
[Setup] Run Tests --randomize suites --randomize tests --randomize
none misc/multiple_suites
@@ -33,11 +45,17 @@
Invalid Option value
Run Should Fail --randomize INVALID ${TESTFILE} Option '--randomize'
does not support value 'INVALID'.
+Invalid seed value
+ Run Should Fail --randomize all:test ${TESTFILE}
Option '--randomize' does not support value 'all:test'.
+
*** Keywords ***
Check That Default Orders Are Correct
Run Tests ${EMPTY} misc/multiple_suites
Should Be Equal As Strings ${SUITE.suites} ${DEFAULT SUITE ORDER}
Should Be Equal As Strings ${SUITE.suites[0].tests} ${DEFAULT TEST
ORDER}
+
+Get Suites
+ [Return] ${SUITE.suites}
Get Tests
Comment This keyword is needed as there is also one directory suite,
which does not contain tests.
=======================================
--- /src/robot/conf/settings.py Thu Jan 30 09:28:17 2014 UTC
+++ /src/robot/conf/settings.py Tue Apr 22 14:15:18 2014 UTC
@@ -158,8 +158,16 @@
formatted_value = original_value.lower()
if formatted_value in ('test', 'suite'):
formatted_value += 's'
- if formatted_value not in ('tests', 'suites', 'none', 'all'):
+ formatted_value = formatted_value.split(':')
+ if len(formatted_value) not in (1,2):
self._raise_invalid_option_value('--randomize', original_value)
+ if formatted_value[0] not in ('tests', 'suites', 'none', 'all'):
+ self._raise_invalid_option_value('--randomize', original_value)
+ if len(formatted_value) == 2:
+ try:
+ formatted_value[1] = int(formatted_value[1])
+ except ValueError:
+ self._raise_invalid_option_value('--randomize',
original_value)
return formatted_value
def _raise_invalid_option_value(self, option_name, given_value):
@@ -415,17 +423,27 @@
'include_tests': self['TestNames'],
'empty_suite_ok': self['RunEmptySuite'],
'randomize_suites': self.randomize_suites,
- 'randomize_tests': self.randomize_tests
+ 'randomize_tests': self.randomize_tests,
+ 'randomize_seed': self.randomize_seed,
}
+
+ @property
+ def randomize_seed(self):
+ randomize_opts = self['Randomize']
+ if len(randomize_opts) == 2:
+ return randomize_opts[1]
+ else:
+ import random
+ return random.randint(1,99999)
@property
def randomize_suites(self):
- return (self['Randomize'] in ('suites', 'all') or
+ return (self['Randomize'][0] in ('suites', 'all') or
any(mode in ('random:suite', 'random:all') for mode in
self['RunMode']))
@property
def randomize_tests(self):
- return (self['Randomize'] in ('tests', 'all') or
+ return (self['Randomize'][0] in ('tests', 'all') or
any(mode in ('random:test', 'random:all') for mode in
self['RunMode']))
@property
=======================================
--- /src/robot/run.py Thu Jan 30 09:28:17 2014 UTC
+++ /src/robot/run.py Tue Apr 22 14:15:18 2014 UTC
@@ -261,9 +261,10 @@
--skipteardownonexit Causes teardowns to be skipped if test execution
is
stopped prematurely.
--randomize all|suites|tests|none Randomizes the test execution order.
- all: randomizes both suites and tests
- suites: randomizes suites
- tests: randomizes tests
+ all[:<seed>]: randomizes both suites and
tests with
+ optional seed
+ suites[:<seed>]: randomizes suites with optional
seed
+ tests[:<seed>]: randomizes tests with optional
seed
none: no randomization (default)
--runmode mode * Deprecated in version 2.8. Use individual options
--dryrun, --exitonfailure, --skipteardownonexit,
or
=======================================
--- /src/robot/running/model.py Sun Jan 26 16:46:11 2014 UTC
+++ /src/robot/running/model.py Tue Apr 22 14:15:18 2014 UTC
@@ -126,14 +126,14 @@
def variables(self, variables):
return model.ItemList(Variable, {'source': self.source},
items=variables)
- def configure(self, randomize_suites=False, randomize_tests=False,
+ def configure(self, randomize_suites=False, randomize_tests=False,
randomize_seed=None,
**options):
model.TestSuite.configure(self, **options)
- self.randomize(randomize_suites, randomize_tests)
+ self.randomize(randomize_suites, randomize_tests, randomize_seed)
- def randomize(self, suites=True, tests=True):
+ def randomize(self, suites=True, tests=True, seed=None):
"""Randomizes the order of suites and/or tests, recursively."""
- self.visit(Randomizer(suites, tests))
+ self.visit( Randomizer(suites, tests, seed) )
def run(self, settings=None, **options):
"""Executes the suite based based the given ``settings`` or
``options``.
=======================================
--- /src/robot/running/randomizer.py Thu Jan 23 14:00:53 2014 UTC
+++ /src/robot/running/randomizer.py Tue Apr 22 14:15:18 2014 UTC
@@ -19,9 +19,11 @@
class Randomizer(SuiteVisitor):
- def __init__(self, randomize_suites=True, randomize_tests=True):
+ def __init__(self, randomize_suites=True, randomize_tests=True,
seed=None):
self.randomize_suites = randomize_suites
self.randomize_tests = randomize_tests
+ if seed is not None:
+ random.seed( seed )
def start_suite(self, suite):
if not self.randomize_suites and not self.randomize_tests:
=======================================
--- /utest/running/test_randomizer.py Fri May 31 11:26:14 2013 UTC
+++ /utest/running/test_randomizer.py Tue Apr 22 14:15:18 2014 UTC
@@ -7,9 +7,13 @@
names = [str(i) for i in range(100)]
def setUp(self):
- self.suite = TestSuite()
- self.suite.suites = self._generate_suites()
- self.suite.tests = self._generate_tests()
+ self.suite = self._generate_suite()
+
+ def _generate_suite(self):
+ s = TestSuite()
+ s.suites = self._generate_suites()
+ s.tests = self._generate_tests()
+ return s
def _generate_suites(self):
return [TestSuite(name=n) for n in self.names]
@@ -61,6 +65,23 @@
assert_equals([t.id for t in self.suite.tests],
['s1-t%d' % i for i in range(1, 101)])
+ def _gen_random_suite(self, seed):
+ suite = self._generate_suite()
+ suite.randomize(suites=True, tests=True, seed=seed)
+ random_order_suites = [i.name for i in suite.suites]
+ random_order_tests = [i.name for i in suite.tests]
+ return (random_order_suites, random_order_tests)
+
+ def test_randomize_seed(self):
+ """
+ GIVEN a test suite
+ WHEN it's randomized with a given seed
+ THEN it's always sorted in the same order
+ """
+ (random_order_suites1, random_order_tests1) =
self._gen_random_suite(1234)
+ (random_order_suites2, random_order_tests2) =
self._gen_random_suite(1234)
+ assert_equals( random_order_suites1, random_order_suites2 )
+ assert_equals( random_order_tests1, random_order_tests2 )
if __name__ == '__main__':
unittest.main()
==============================================================================
Revision: 14bbd457f8dc
Branch: default
Author: Pekka Klärck
Date: Tue Apr 22 15:56:54 2014 UTC
Log: Clean-up and fix giving seed to --randomize.
Update issue 1673
Some clean-up and fixes:
- Cleaned up acceptance tests a little. Including old tests for randomizing.
- Added separate tests for randomizing only suites/tests. Also tested using
both 'suites/tests' and 'suite/test' variants. I think the versions
without 's' were originally supported because that was the syntax used with
older '--runmode' option.
- Cleaned up parsing option value. This includes fix for
handling 'test:1234' and 'suite:5678'.
- Style: Removed whitespace inside parenthesis, cut lines over 80 chars.
- Enhanced --help documentation.
http://code.google.com/p/robotframework/source/detail?r=14bbd457f8dc
Modified:
/atest/robot/cli/runner/randomize.txt
/src/robot/conf/settings.py
/src/robot/run.py
/src/robot/running/model.py
/src/robot/running/randomizer.py
=======================================
--- /atest/robot/cli/runner/randomize.txt Tue Apr 22 14:15:18 2014 UTC
+++ /atest/robot/cli/runner/randomize.txt Tue Apr 22 15:56:54 2014 UTC
@@ -8,41 +8,57 @@
${DEFAULT TEST ORDER} [test1, test2, test3, test4, test5, test6, test7,
test8, test9, test10, test11, test12]
*** Test Cases ***
-Randomizing Tests
+Randomizing tests
[Setup] Run Tests --randomize test
misc/multiple_suites/01__suite_first.html
Should Not Be Equal As Strings ${SUITE.tests} ${DEFAULT TEST ORDER}
-Randomizing Suites
+Randomizing suites
[Setup] Run Tests --randomize Suites misc/multiple_suites
- Should Not Be Equal As Strings ${SUITE.suites} ${DEFAULT SUITE ORDER}
- ${tests} = Get Tests
- Should Be Equal As Strings ${tests} ${DEFAULT TEST ORDER}
+ Suites should be randomized
+ Tests should be in default order
-Randomizing Suites And Tests
+Randomizing suites and tests
[Setup] Run Tests --randomize all misc/multiple_suites
- Should Not Be Equal As Strings ${SUITE.suites} ${DEFAULT SUITE ORDER}
- ${tests} = Get Tests
- Should Not Be Equal As Strings ${tests} ${DEFAULT TEST ORDER}
+ Suites should be randomized
+ Tests should be randomized
-Randomizing Suites And Tests with seed
- [Setup] Run Tests --randomize all:1234 misc/multiple_suites
- ${suites1} = Get Suites
- ${tests1} = Get Tests
- Should Not Be Equal As Strings ${suites1} ${DEFAULT SUITE ORDER}
- Should Not Be Equal As Strings ${tests1} ${DEFAULT TEST ORDER}
+Randomizing tests with seed
+ Run Tests --randomize test:1234 misc/multiple_suites
+ ${suites1} = Suites should be in default order
+ ${tests1} = Tests should be randomized
+ Run Tests --randomize TESTS:1234 misc/multiple_suites
+ ${suites2} = Suites should be in default order
+ ${tests2} = Tests should be randomized
+ Order should be same ${suites1} ${suites2}
+ Order should be same ${tests1} ${tests2}
+
+Randomizing suites with seed
+ Run Tests --randomize Suite:1234 misc/multiple_suites
+ ${suites1} = Suites should be randomized
+ ${tests1} = Tests should be in default order
+ Run Tests --randomize SuiteS:1234 misc/multiple_suites
+ ${suites2} = Suites should be randomized
+ ${tests2} = Tests should be in default order
+ Order should be same ${suites1} ${suites2}
+ Order should be same ${tests1} ${tests2}
+
+Randomizing suites and tests with seed
Run Tests --randomize all:1234 misc/multiple_suites
- ${suites2} = Get Suites
- ${tests2} = Get Tests
- Should Be Equal As Strings ${suites1} ${suites2}
- Should Be Equal As Strings ${tests1} ${tests2}
+ ${suites1} = Suites should be randomized
+ ${tests1} = Tests should be randomized
+ Run Tests --randomize ALL:1234 misc/multiple_suites
+ ${suites2} = Suites should be randomized
+ ${tests2} = Tests should be randomized
+ Order should be same ${suites1} ${suites2}
+ Order should be same ${tests1} ${tests2}
-Last Option Overrides All Previous
+Last option overrides all previous
[Setup] Run Tests --randomize suites --randomize tests --randomize
none misc/multiple_suites
Should Be Equal As Strings ${SUITE.suites} ${DEFAULT SUITE ORDER}
${tests} = Get Tests
Should Be Equal As Strings ${tests} ${DEFAULT TEST ORDER}
-Invalid Option value
+Invalid option value
Run Should Fail --randomize INVALID ${TESTFILE} Option '--randomize'
does not support value 'INVALID'.
Invalid seed value
@@ -51,13 +67,32 @@
*** Keywords ***
Check That Default Orders Are Correct
Run Tests ${EMPTY} misc/multiple_suites
- Should Be Equal As Strings ${SUITE.suites} ${DEFAULT SUITE ORDER}
- Should Be Equal As Strings ${SUITE.suites[0].tests} ${DEFAULT TEST
ORDER}
+ Suites should be in default order
+ Tests should be in default order
+
+Suites Should Be Randomized
+ Should Not Be Equal As Strings ${SUITE.suites} ${DEFAULT SUITE
ORDER}
+ [Return] ${SUITE.suites}
-Get Suites
+Suites should be in default order
+ Should Be Equal As Strings ${SUITE.suites} ${DEFAULT SUITE ORDER}
[Return] ${SUITE.suites}
+
+Tests Should Be Randomized
+ ${tests} = Get Tests
+ Should Not Be Equal As Strings ${tests} ${DEFAULT TEST ORDER}
+ [Return] ${tests}
+
+Tests should be in default order
+ ${tests} = Get Tests
+ Should Be Equal As Strings ${tests} ${DEFAULT TEST ORDER}
+ [Return] ${tests}
+
+Order should be same
+ [Arguments] ${first} ${second}
+ Should Be Equal As Strings ${first} ${second}
Get Tests
- Comment This keyword is needed as there is also one directory suite,
which does not contain tests.
+ # This keyword is needed because 'Sub.Suite.1' is directory and thus
doesn't itself have tests
${tests} = Set Variable If '${SUITE.suites[0].name}'
== 'Sub.Suite.1' ${SUITE.suites[0].suites[0].tests}
${SUITE.suites[0].tests}
[Return] ${tests}
=======================================
--- /src/robot/conf/settings.py Tue Apr 22 14:15:18 2014 UTC
+++ /src/robot/conf/settings.py Tue Apr 22 15:56:54 2014 UTC
@@ -13,6 +13,8 @@
# limitations under the License.
import os
+import random
+import sys
from robot import utils
from robot.errors import DataError, FrameworkError
@@ -154,25 +156,25 @@
raise DataError("Default visible log level '%s' is lower than "
"log level '%s'" % (default, log_level))
- def _process_randomize_value(self, original_value):
- formatted_value = original_value.lower()
- if formatted_value in ('test', 'suite'):
- formatted_value += 's'
- formatted_value = formatted_value.split(':')
- if len(formatted_value) not in (1,2):
- self._raise_invalid_option_value('--randomize', original_value)
- if formatted_value[0] not in ('tests', 'suites', 'none', 'all'):
- self._raise_invalid_option_value('--randomize', original_value)
- if len(formatted_value) == 2:
- try:
- formatted_value[1] = int(formatted_value[1])
- except ValueError:
- self._raise_invalid_option_value('--randomize',
original_value)
- return formatted_value
+ def _process_randomize_value(self, original):
+ value = original.lower()
+ if ':' in value:
+ value, seed = value.split(':', 1)
+ else:
+ seed = random.randint(0, sys.maxint)
+ if value in ('test', 'suite'):
+ value += 's'
+ if value not in ('tests', 'suites', 'none', 'all'):
+ self._raise_invalid_option_value('--randomize', original)
+ try:
+ seed = int(seed)
+ except ValueError:
+ self._raise_invalid_option_value('--randomize', original)
+ return value, seed
def _raise_invalid_option_value(self, option_name, given_value):
- raise DataError("Option '%s' does not support value '%s'." %
- (option_name, given_value))
+ raise DataError("Option '%s' does not support value '%s'."
+ % (option_name, given_value))
def _process_runmode_value(self, original_value):
formatted_value = original_value.lower()
@@ -429,12 +431,7 @@
@property
def randomize_seed(self):
- randomize_opts = self['Randomize']
- if len(randomize_opts) == 2:
- return randomize_opts[1]
- else:
- import random
- return random.randint(1,99999)
+ return self['Randomize'][1]
@property
def randomize_suites(self):
=======================================
--- /src/robot/run.py Tue Apr 22 14:15:18 2014 UTC
+++ /src/robot/run.py Tue Apr 22 15:56:54 2014 UTC
@@ -261,11 +261,13 @@
--skipteardownonexit Causes teardowns to be skipped if test execution
is
stopped prematurely.
--randomize all|suites|tests|none Randomizes the test execution order.
- all[:<seed>]: randomizes both suites and
tests with
- optional seed
- suites[:<seed>]: randomizes suites with optional
seed
- tests[:<seed>]: randomizes tests with optional
seed
+ all: randomizes both suites and tests
+ suites: randomizes suites
+ tests: randomizes tests
none: no randomization (default)
+ Use syntax `VALUE:SEED` to use a custom random
seed.
+ Examples: --randomize all
+ --randomize tests:1234
--runmode mode * Deprecated in version 2.8. Use individual options
--dryrun, --exitonfailure, --skipteardownonexit,
or
--randomize instead.
=======================================
--- /src/robot/running/model.py Tue Apr 22 14:15:18 2014 UTC
+++ /src/robot/running/model.py Tue Apr 22 15:56:54 2014 UTC
@@ -126,14 +126,14 @@
def variables(self, variables):
return model.ItemList(Variable, {'source': self.source},
items=variables)
- def configure(self, randomize_suites=False, randomize_tests=False,
randomize_seed=None,
- **options):
+ def configure(self, randomize_suites=False, randomize_tests=False,
+ randomize_seed=None, **options):
model.TestSuite.configure(self, **options)
self.randomize(randomize_suites, randomize_tests, randomize_seed)
def randomize(self, suites=True, tests=True, seed=None):
"""Randomizes the order of suites and/or tests, recursively."""
- self.visit( Randomizer(suites, tests, seed) )
+ self.visit(Randomizer(suites, tests, seed))
def run(self, settings=None, **options):
"""Executes the suite based based the given ``settings`` or
``options``.
=======================================
--- /src/robot/running/randomizer.py Tue Apr 22 14:15:18 2014 UTC
+++ /src/robot/running/randomizer.py Tue Apr 22 15:56:54 2014 UTC
@@ -23,7 +23,7 @@
self.randomize_suites = randomize_suites
self.randomize_tests = randomize_tests
if seed is not None:
- random.seed( seed )
+ random.seed(seed)
def start_suite(self, suite):
if not self.randomize_suites and not self.randomize_tests:
--
---
You received this message because you are subscribed to the Google Groups "robotframework-commit" group.
To unsubscribe from this group and stop receiving emails from it, send an email
to robotframework-commit+unsubscr...@googlegroups.com.
For more options, visit https://groups.google.com/d/optout.
