jenkins-bot has submitted this change and it was merged. ( https://gerrit.wikimedia.org/r/323540 )
Change subject: Support skin as dependencies ...................................................................... Support skin as dependencies Rename Zuul method set_ext_dependencies to a more generic set_mw_dependencies Dependent skins are injected via a new SKIN_DEPENDENCIES variable. We only need to clone the repositories, the MediaWiki installer takes care of injecting/installing them in LocalSettings.php unlike extensions that rely on extensions_load.txt. Refactor/enhance tests to take skins in account. Add a few tests for get_dependencies() Bug: T151593 Signed-off-by: Antoine Musso <[email protected]> Change-Id: Icad88b9b0995199eb3e75b9e5ce9f7a0acfee16e --- M jjb/mediawiki-extensions.yaml D tests/test_zuul_ext_dependencies.py A tests/test_zuul_mw_dependencies.py M zuul/parameter_functions.py 4 files changed, 192 insertions(+), 94 deletions(-) Approvals: Paladox: Looks good to me, but someone else must approve Hashar: Looks good to me, approved jenkins-bot: Verified diff --git a/jjb/mediawiki-extensions.yaml b/jjb/mediawiki-extensions.yaml index 69b69d4..1ece20c 100644 --- a/jjb/mediawiki-extensions.yaml +++ b/jjb/mediawiki-extensions.yaml @@ -66,11 +66,16 @@ builders: - shell: "echo $ZUUL_PROJECT > deps.txt" - shell: "echo -e $EXT_DEPENDENCIES >> deps.txt" + - shell: "echo -e $SKIN_DEPENDENCIES > deps_skins.txt" + # Clone both extensions and skins, but only extensions get listed for the + # extensions autoloader in integration/jenkins.git, skins are + # automatically injected by MediaWiki upon installation. - zuul-cloner: projects: > mediawiki/core mediawiki/vendor $(cat deps.txt) + $(cat deps_skins.txt) - ve-submodules-update - shell: "mv deps.txt src/extensions_load.txt" - mw-install-mysql @@ -83,10 +88,12 @@ builders: - shell: "echo $ZUUL_PROJECT > deps.txt" - shell: "echo -e $EXT_DEPENDENCIES >> deps.txt" + - shell: "echo -e $SKIN_DEPENDENCIES > deps_skins.txt" - zuul-cloner: projects: > mediawiki/core $(cat deps.txt) + $(cat deps_skins.txt) - ve-submodules-update - shell: "mv deps.txt src/extensions_load.txt" - composer-validate: diff --git a/tests/test_zuul_ext_dependencies.py b/tests/test_zuul_ext_dependencies.py deleted file mode 100644 index d9c798f..0000000 --- a/tests/test_zuul_ext_dependencies.py +++ /dev/null @@ -1,73 +0,0 @@ -import os -import unittest - -from fakes import FakeJob - -dependencies = {} # defined for flake8 -get_dependencies = None # defined for flake8 -set_parameters = None # defined for flake8 -# Import function -execfile(os.path.join( - os.path.dirname(os.path.abspath(__file__)), - '../zuul/parameter_functions.py')) - - -class TestExtDependencies(unittest.TestCase): - def assertHasDependencies(self, params): - self.assertIn('EXT_DEPENDENCIES', params) - - def assertMissingDependencies(self, params): - self.assertNotIn('EXT_DEPENDENCIES', params) - - def fetch_dependencies(self, job_name=None, project=None): - if project: - params = {'ZUUL_PROJECT': project} - else: - params = {'ZUUL_PROJECT': 'mediawiki/extensions/Example'} - job = FakeJob(job_name if job_name else 'mwext-testextension-hhvm') - set_parameters(None, job, params) - return params - - def test_ext_name(self): - params = self.fetch_dependencies( - project='mediawiki/extensions/Example') - - self.assertIn('EXT_NAME', params) - self.assertEqual(params['EXT_NAME'], 'Example') - - def test_cyclical_dependencies(self): - """verifies that cyclical dependencies are possible""" - - mapping = {'Foo': ['Bar'], 'Bar': ['Foo']} - - self.assertEqual(get_dependencies('Foo', mapping), set(['Foo', 'Bar'])) - - def test_resolvable_dependencies(self): - """verifies that we can resolve all of the dependencies""" - for ext_name in dependencies: - self.assertHasDependencies(self.fetch_dependencies( - project='mediawiki/extensions/' + ext_name)) - - def test_job_name(self): - self.assertHasDependencies(self.fetch_dependencies( - job_name='mwext-testextension-hhvm')) - self.assertHasDependencies(self.fetch_dependencies( - job_name='mwext-qunit-jessie')) - self.assertHasDependencies(self.fetch_dependencies( - job_name='mwext-qunit-composer-jessie')) - self.assertHasDependencies(self.fetch_dependencies( - job_name='mwext-mw-selenium-composer-jessie')) - self.assertHasDependencies(self.fetch_dependencies( - job_name='mwext-mw-selenium-jessie')) - self.assertMissingDependencies(self.fetch_dependencies( - job_name='mediawiki-core-phplint')) - - def test_zuul_project_name(self): - self.assertHasDependencies(self.fetch_dependencies( - project='mediawiki/extensions/Example')) - self.assertMissingDependencies(self.fetch_dependencies( - project='mediawiki/extensions')) - self.assertMissingDependencies(self.fetch_dependencies( - project='mediawiki/extensions/Example/vendor')) - self.assertMissingDependencies(self.fetch_dependencies( - project='foo/bar/baz')) diff --git a/tests/test_zuul_mw_dependencies.py b/tests/test_zuul_mw_dependencies.py new file mode 100644 index 0000000..bc30074 --- /dev/null +++ b/tests/test_zuul_mw_dependencies.py @@ -0,0 +1,130 @@ +import os +import unittest + +from fakes import FakeJob + +dependencies = {} # defined for flake8 +get_dependencies = None # defined for flake8 +set_parameters = None # defined for flake8 +# Import function +execfile(os.path.join( + os.path.dirname(os.path.abspath(__file__)), + '../zuul/parameter_functions.py')) + + +class TestMwDependencies(unittest.TestCase): + def assertHasDependencies(self, params): + self.assertIn('EXT_DEPENDENCIES', params) + + def assertMissingDependencies(self, params): + self.assertNotIn('EXT_DEPENDENCIES', params) + self.assertNotIn('SKIN_DEPENDENCIES', params) + + def fetch_dependencies(self, job_name=None, project=None): + if project: + params = {'ZUUL_PROJECT': project} + else: + params = {'ZUUL_PROJECT': 'mediawiki/extensions/Example'} + job = FakeJob(job_name if job_name else 'mwext-testextension-hhvm') + set_parameters(None, job, params) + return params + + def test_ext_name(self): + params = self.fetch_dependencies( + project='mediawiki/extensions/Example') + + self.assertIn('EXT_NAME', params) + self.assertEqual(params['EXT_NAME'], 'Example') + + def test_skin_name(self): + params = self.fetch_dependencies( + project='mediawiki/skins/Vector') + + self.assertIn('SKIN_NAME', params) + self.assertEqual(params['SKIN_NAME'], 'Vector') + + def test_cyclical_dependencies(self): + """verifies that cyclical dependencies are possible""" + + mapping = {'Foo': ['Bar'], 'Bar': ['Foo']} + + self.assertEqual(get_dependencies('Foo', mapping), set(['Foo', 'Bar'])) + + def test_cyclical_dependencies_with_skins(self): + mapping = {'Foo': ['skins/Vector'], 'skins/Vector': ['Foo']} + self.assertEqual( + get_dependencies('skins/Vector', mapping), + set(['Foo', 'skins/Vector']) + ) + + def test_resolvable_dependencies(self): + """verifies that we can resolve all of the dependencies""" + for base_name in dependencies: + if base_name.startswith('skins/'): + project = 'mediawiki/' + base_name + else: + project = 'mediawiki/extensions/' + base_name + + self.assertHasDependencies(self.fetch_dependencies( + project=project)) + + def test_job_name(self): + self.assertHasDependencies(self.fetch_dependencies( + job_name='mwext-testextension-hhvm')) + self.assertHasDependencies(self.fetch_dependencies( + job_name='mwext-qunit-jessie')) + self.assertHasDependencies(self.fetch_dependencies( + job_name='mwext-qunit-composer-jessie')) + self.assertHasDependencies(self.fetch_dependencies( + job_name='mwext-mw-selenium-composer-jessie')) + self.assertHasDependencies(self.fetch_dependencies( + job_name='mwext-mw-selenium-jessie')) + + self.assertHasDependencies(self.fetch_dependencies( + job_name='mw-testskin')) + self.assertHasDependencies(self.fetch_dependencies( + job_name='mw-testskin-non-voting')) + + self.assertMissingDependencies(self.fetch_dependencies( + job_name='mediawiki-core-phplint')) + + def test_zuul_project_name(self): + self.assertHasDependencies(self.fetch_dependencies( + project='mediawiki/extensions/Example')) + + self.assertMissingDependencies(self.fetch_dependencies( + project='mediawiki/extensions')) + self.assertMissingDependencies(self.fetch_dependencies( + project='mediawiki/skins')) + self.assertMissingDependencies(self.fetch_dependencies( + project='mediawiki/extensions/Example/vendor')) + self.assertMissingDependencies(self.fetch_dependencies( + project='foo/bar/baz')) + + def test_resolve_skin_on_extension(self): + mapping = {'Foo': ['skins/Vector']} + self.assertEqual( + get_dependencies('Foo', mapping), + set(['skins/Vector']) + ) + + def test_resolve_extension_on_skin(self): + mapping = {'skins/Vector': ['Foo']} + self.assertEqual( + get_dependencies('skins/Vector', mapping), + set(['Foo']) + ) + + def test_resolve_extension_on_extension(self): + mapping = {'Foo': ['DepExtension']} + self.assertEqual( + get_dependencies('Foo', mapping), + set(['DepExtension']) + ) + + def test_resolve_skin_on_skin(self): + mapping = {'skins/Child': ['skin/Common']} + self.assertEqual( + get_dependencies('skins/Child', mapping), + set(['skin/Common']) + ) diff --git a/zuul/parameter_functions.py b/zuul/parameter_functions.py index a716f0e..7e5fbb9 100644 --- a/zuul/parameter_functions.py +++ b/zuul/parameter_functions.py @@ -49,13 +49,15 @@ # T128091: oojs/ui npm job runs on Jessie which only has HHVM params['PHP_BIN'] = 'hhvm' - ext_deps_jobs_starting_with = ( + mw_deps_jobs_starting_with = ( 'mwext-testextension', 'mwext-qunit', 'mwext-mw-selenium', + 'mw-testskin', + 'mw-testskin-non-voting', ) - if job.name.startswith(ext_deps_jobs_starting_with): - set_ext_dependencies(item, job, params) + if job.name.startswith(mw_deps_jobs_starting_with): + set_mw_dependencies(item, job, params) if job.name.startswith('mediawiki-extensions-'): set_gated_extensions(item, job, params) @@ -93,6 +95,11 @@ params['BUILD_TIMEOUT'] = 180 # minutes +# This has is used to inject dependencies for MediaWiki jobs. +# +# Values are assumed to be MediaWiki extensions. Skins have to be prefixed with +# 'skins/'. The has is used by the set_mw_dependencies() parameter function +# below. dependencies = { 'AbuseFilter': ['AntiSpoof'], 'ApiFeatureUsage': ['Elastica'], @@ -194,39 +201,66 @@ } -def set_ext_dependencies(item, job, params): +def set_mw_dependencies(item, job, params): """ - Reads dependencies from the yaml file and adds them as a parameter + Inject MediaWiki dependencies based on a built-in hash. + + Reads MediaWiki dependencies for a repository and inject them as + parameters EXT_DEPENDENCIES or SKIN_DEPENDENCIES. The map is configured via + the 'dependencies' dictionary above. + + Extensions and skins will both be cloned, the extensions will be listed for + the extensions autoloader in integration/jenkins.git, skins are + automatically injected by MediaWiki upon installation. + :type item: zuul.model.QueueItem :type job: zuul.model.Job :type params: dict """ - if not params['ZUUL_PROJECT'].startswith('mediawiki/extensions/'): + if not params['ZUUL_PROJECT'].startswith(( + 'mediawiki/extensions/', + 'mediawiki/skins/', + )): return - # mediawiki/extensions/FooBar + split = params['ZUUL_PROJECT'].split('/') + if len(split) != 3: - # mediawiki/extensions/FooBar/blah - # mediawiki/extensions return - # FooBar - ext_name = split[-1] - params['EXT_NAME'] = ext_name + if split[1] == 'skins': + # Lookup key in 'dependencies'. Example: 'skins/Vector' + dep_key = 'skins' + '/' + split[-1] + # 'Vector' + params['SKIN_NAME'] = split[-1] + else: + # Lookup key in 'dependencies. Example: 'Foobar' + dep_key = split[-1] + params['EXT_NAME'] = split[-1] - deps = get_dependencies(ext_name, dependencies) + deps = get_dependencies(dep_key, dependencies) - # Export with a literal \n character and have bash expand it later - params['EXT_DEPENDENCIES'] = '\\n'.join( - 'mediawiki/extensions/' + dep for dep in sorted(deps) - ) + # Split extensions and skins + skin_deps = {d for d in deps if d.startswith('skins/')} + ext_deps = deps - skin_deps + + # Export with a literal \n character and have bash expand it later via + # 'echo -e $XXX_DEPENDENCIES'. + def glue_deps(prefix, deps): + return '\\n'.join( + prefix + d for d in sorted(deps) + ) + + params['SKIN_DEPENDENCIES'] = glue_deps('mediawiki/', skin_deps) + params['EXT_DEPENDENCIES'] = glue_deps('mediawiki/extensions/', ext_deps) -def get_dependencies(ext_name, mapping): +def get_dependencies(key, mapping): """ Get the full set of dependencies required by an extension - :param ext_name: extension name - :param mapping: mapping of extensions to their dependencies + + :param key: extension base name or skin as 'skin/BASENAME' + :param mapping: mapping of repositories to their dependencies :return: set of dependencies, recursively processed """ resolved = set() @@ -244,7 +278,7 @@ return deps - return resolve_deps(ext_name) + return resolve_deps(key) gatedextensions = [ -- To view, visit https://gerrit.wikimedia.org/r/323540 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: merged Gerrit-Change-Id: Icad88b9b0995199eb3e75b9e5ce9f7a0acfee16e Gerrit-PatchSet: 20 Gerrit-Project: integration/config Gerrit-Branch: master Gerrit-Owner: Paladox <[email protected]> Gerrit-Reviewer: Hashar <[email protected]> Gerrit-Reviewer: Legoktm <[email protected]> Gerrit-Reviewer: Paladox <[email protected]> Gerrit-Reviewer: Zfilipin <[email protected]> Gerrit-Reviewer: jenkins-bot <> _______________________________________________ MediaWiki-commits mailing list [email protected] https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
