Volans has uploaded a new change for review. (
https://gerrit.wikimedia.org/r/363747 )
Change subject: Configuration: automatically load backend's aliases
......................................................................
Configuration: automatically load backend's aliases
* when loading the configuration, automatically load also any backend's
aliases file present in the same directory of the configuration file.
Support for aliases into the backends will be added next.
* improved tests fixture usage and removed usage of the example
configuration present in the documentation from the tests.
Bug: T169640
Change-Id: Ief4c374408741af0e29d9bf07d8976b8f753d78d
---
M cumin/__init__.py
M cumin/tests/__init__.py
A cumin/tests/fixtures/config/empty/config.yaml
A cumin/tests/fixtures/config/invalid/config.yaml
A cumin/tests/fixtures/config/valid/config.yaml
A cumin/tests/fixtures/config/valid_with_aliases/config.yaml
A cumin/tests/fixtures/config/valid_with_aliases/direct_aliases.yaml
A cumin/tests/fixtures/config/valid_with_aliases/puppetdb_aliases.yaml
A cumin/tests/fixtures/config/valid_with_empty_aliases/config.yaml
A cumin/tests/fixtures/config/valid_with_empty_aliases/direct_aliases.yaml
A cumin/tests/fixtures/config/valid_with_empty_aliases/puppetdb_aliases.yaml
A cumin/tests/fixtures/config/valid_with_invalid_aliases/config.yaml
A cumin/tests/fixtures/config/valid_with_invalid_aliases/puppetdb_aliases.yaml
R cumin/tests/fixtures/grammar/invalid_grammars.txt
R cumin/tests/fixtures/grammar/valid_grammars.txt
M cumin/tests/unit/test_grammar.py
M cumin/tests/unit/test_init.py
17 files changed, 156 insertions(+), 24 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/operations/software/cumin
refs/changes/47/363747/1
diff --git a/cumin/__init__.py b/cumin/__init__.py
index 941208c..7d450dd 100644
--- a/cumin/__init__.py
+++ b/cumin/__init__.py
@@ -1,4 +1,7 @@
"""Automation and orchestration framework written in Python."""
+import os
+import pkgutil
+
from pkg_resources import DistributionNotFound, get_distribution
import yaml
@@ -29,6 +32,7 @@
"""
if config not in cls._instances:
cls._instances[config] = parse_config(config)
+ load_backend_aliases(cls._instances[config],
os.path.dirname(config))
return cls._instances[config]
@@ -52,3 +56,26 @@
raise CuminError("Empty configuration found in
'{config}'".format(config=config_file))
return config
+
+
+def load_backend_aliases(config, base_path):
+ """Given a configuration, automatically add backend aliases from
configuration files in the base_path directory.
+
+ It will look for files named {backend}_aliases.yaml in the base_path
directory and will load it's content into the
+ main configuration under the 'aliases' key under the backend specific
configuration, so that it will be accessible
+ by config[backend]['aliases'].
+
+ Arguments:
+ config -- the configuration object to add the aliases to.
+ base_path -- the base path where to look for the aliases files.
+ """
+ abs_path = os.path.dirname(os.path.abspath(__file__))
+ backends = [name for _, name, _ in
pkgutil.iter_modules([os.path.join(abs_path, 'backends')])]
+
+ for backend in backends:
+ alias_file = os.path.join(base_path,
'{backend}_aliases.yaml'.format(backend=backend))
+ if os.path.isfile(alias_file): # Do not fail if the alias file
doesn't exists
+ if config.get(backend) is None:
+ config[backend] = {}
+
+ config[backend]['aliases'] = parse_config(alias_file)
diff --git a/cumin/tests/__init__.py b/cumin/tests/__init__.py
index 742e027..de0b2a3 100644
--- a/cumin/tests/__init__.py
+++ b/cumin/tests/__init__.py
@@ -7,17 +7,26 @@
_TESTS_BASE_PATH = os.path.realpath(os.path.dirname(__file__))
-def get_fixture(filename, as_string=False):
+def get_fixture(path, as_string=False):
"""Return the content of a fixture file.
Arguments:
- filename -- the file to be opened in the test's fixture directory
+ path -- the relative path to the test's fixture directory to be
opened.
as_string -- return the content as a multiline string instead of a list of
lines [optional, default: False]
"""
- with open(os.path.join(_TESTS_BASE_PATH, 'fixtures', filename)) as f:
+ with open(get_fixture_path(path)) as f:
if as_string:
content = f.read()
else:
content = f.readlines()
return content
+
+
+def get_fixture_path(path):
+ """Return the absolute path of the given fixture.
+
+ Arguments:
+ path -- the relative path to the test's fixture directory.
+ """
+ return os.path.join(_TESTS_BASE_PATH, 'fixtures', path)
diff --git a/cumin/tests/fixtures/config/empty/config.yaml
b/cumin/tests/fixtures/config/empty/config.yaml
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/cumin/tests/fixtures/config/empty/config.yaml
diff --git a/cumin/tests/fixtures/config/invalid/config.yaml
b/cumin/tests/fixtures/config/invalid/config.yaml
new file mode 100644
index 0000000..64c7e22
--- /dev/null
+++ b/cumin/tests/fixtures/config/invalid/config.yaml
@@ -0,0 +1,3 @@
+key:
+ other_key: value
+ - invalid_item
diff --git a/cumin/tests/fixtures/config/valid/config.yaml
b/cumin/tests/fixtures/config/valid/config.yaml
new file mode 100644
index 0000000..b3591c0
--- /dev/null
+++ b/cumin/tests/fixtures/config/valid/config.yaml
@@ -0,0 +1,17 @@
+backend: puppetdb
+transport: clustershell
+log_file: logs/cumin.log
+
+environment:
+ ENV_VARIABLE: env_value
+
+puppetdb:
+ host: puppetdb.local
+ port: 443
+ urllib3_disable_warnings:
+ - SubjectAltNameWarning
+
+clustershell:
+ ssh_options:
+ - 'some_option'
+ fanout: 16
diff --git a/cumin/tests/fixtures/config/valid_with_aliases/config.yaml
b/cumin/tests/fixtures/config/valid_with_aliases/config.yaml
new file mode 120000
index 0000000..579f660
--- /dev/null
+++ b/cumin/tests/fixtures/config/valid_with_aliases/config.yaml
@@ -0,0 +1 @@
+../valid/config.yaml
\ No newline at end of file
diff --git a/cumin/tests/fixtures/config/valid_with_aliases/direct_aliases.yaml
b/cumin/tests/fixtures/config/valid_with_aliases/direct_aliases.yaml
new file mode 100644
index 0000000..10aef29
--- /dev/null
+++ b/cumin/tests/fixtures/config/valid_with_aliases/direct_aliases.yaml
@@ -0,0 +1,3 @@
+group1: host10[10-22].example.org
+group2: host20[10-22].example.org
+group_all: A:group1 OR A:group2
diff --git
a/cumin/tests/fixtures/config/valid_with_aliases/puppetdb_aliases.yaml
b/cumin/tests/fixtures/config/valid_with_aliases/puppetdb_aliases.yaml
new file mode 100644
index 0000000..a98bad3
--- /dev/null
+++ b/cumin/tests/fixtures/config/valid_with_aliases/puppetdb_aliases.yaml
@@ -0,0 +1,3 @@
+role1: R:Class = Role::Role1
+dc1: '*.dc1.example.org'
+group_all: A:role1 AND A:dc1
diff --git a/cumin/tests/fixtures/config/valid_with_empty_aliases/config.yaml
b/cumin/tests/fixtures/config/valid_with_empty_aliases/config.yaml
new file mode 120000
index 0000000..579f660
--- /dev/null
+++ b/cumin/tests/fixtures/config/valid_with_empty_aliases/config.yaml
@@ -0,0 +1 @@
+../valid/config.yaml
\ No newline at end of file
diff --git
a/cumin/tests/fixtures/config/valid_with_empty_aliases/direct_aliases.yaml
b/cumin/tests/fixtures/config/valid_with_empty_aliases/direct_aliases.yaml
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/cumin/tests/fixtures/config/valid_with_empty_aliases/direct_aliases.yaml
diff --git
a/cumin/tests/fixtures/config/valid_with_empty_aliases/puppetdb_aliases.yaml
b/cumin/tests/fixtures/config/valid_with_empty_aliases/puppetdb_aliases.yaml
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/cumin/tests/fixtures/config/valid_with_empty_aliases/puppetdb_aliases.yaml
diff --git a/cumin/tests/fixtures/config/valid_with_invalid_aliases/config.yaml
b/cumin/tests/fixtures/config/valid_with_invalid_aliases/config.yaml
new file mode 120000
index 0000000..579f660
--- /dev/null
+++ b/cumin/tests/fixtures/config/valid_with_invalid_aliases/config.yaml
@@ -0,0 +1 @@
+../valid/config.yaml
\ No newline at end of file
diff --git
a/cumin/tests/fixtures/config/valid_with_invalid_aliases/puppetdb_aliases.yaml
b/cumin/tests/fixtures/config/valid_with_invalid_aliases/puppetdb_aliases.yaml
new file mode 100644
index 0000000..0bc3a7c
--- /dev/null
+++
b/cumin/tests/fixtures/config/valid_with_invalid_aliases/puppetdb_aliases.yaml
@@ -0,0 +1,2 @@
+name: query
+- invalid_name
diff --git a/cumin/tests/fixtures/invalid_grammars.txt
b/cumin/tests/fixtures/grammar/invalid_grammars.txt
similarity index 100%
rename from cumin/tests/fixtures/invalid_grammars.txt
rename to cumin/tests/fixtures/grammar/invalid_grammars.txt
diff --git a/cumin/tests/fixtures/valid_grammars.txt
b/cumin/tests/fixtures/grammar/valid_grammars.txt
similarity index 100%
rename from cumin/tests/fixtures/valid_grammars.txt
rename to cumin/tests/fixtures/grammar/valid_grammars.txt
diff --git a/cumin/tests/unit/test_grammar.py b/cumin/tests/unit/test_grammar.py
index 2c78319..b006fd3 100644
--- a/cumin/tests/unit/test_grammar.py
+++ b/cumin/tests/unit/test_grammar.py
@@ -1,5 +1,7 @@
"""Grammar tests."""
+import os
+
from cumin.grammar import grammar
from cumin.tests import get_fixture
@@ -13,13 +15,14 @@
def test_valid_strings():
"""Run quick pyparsing test over valid grammar strings."""
- results = grammar.runTests(get_fixture('valid_grammars.txt',
as_string=True))
+ results = grammar.runTests(get_fixture(os.path.join('grammar',
'valid_grammars.txt'), as_string=True))
assert results[0]
def test_invalid_strings():
"""Run quick pyparsing test over invalid grammar strings."""
- results = grammar.runTests(get_fixture('invalid_grammars.txt',
as_string=True), failureTests=True)
+ results = grammar.runTests(
+ get_fixture(os.path.join('grammar', 'invalid_grammars.txt'),
as_string=True), failureTests=True)
assert results[0]
diff --git a/cumin/tests/unit/test_init.py b/cumin/tests/unit/test_init.py
index 12a9608..c2b9e3f 100644
--- a/cumin/tests/unit/test_init.py
+++ b/cumin/tests/unit/test_init.py
@@ -1,23 +1,62 @@
"""Cumin package tests."""
+# pylint: disable=invalid-name
import os
-import tempfile
import pytest
import cumin
+from cumin.tests import get_fixture_path
-def test_config_class_instantiation():
+
+def test_config_class_valid():
"""Should return the config. Multiple Config with the same path should
return the same object."""
- config1 = cumin.Config('doc/examples/config.yaml')
+ config_file = get_fixture_path(os.path.join('config', 'valid',
'config.yaml'))
+ config1 = cumin.Config(config=config_file)
assert 'log_file' in config1
- config2 = cumin.Config('doc/examples/config.yaml')
+ config2 = cumin.Config(config=config_file)
assert config1 is config2
+
+
+def test_config_class_empty():
+ """A CuminError is raised if the configuration is empty."""
+ with pytest.raises(cumin.CuminError, match='Empty configuration found in'):
+ cumin.Config(config=get_fixture_path(os.path.join('config', 'empty',
'config.yaml')))
+
+
+def test_config_class_invalid():
+ """A CuminError is raised if the configuration cannot be parsed."""
+ with pytest.raises(cumin.CuminError, match='Unable to parse configuration
file'):
+ cumin.Config(config=get_fixture_path(os.path.join('config', 'invalid',
'config.yaml')))
+
+
+def test_config_class_valid_with_aliases():
+ """Should return the config including the backend aliases."""
+ config = cumin.Config(config=get_fixture_path(os.path.join('config',
'valid_with_aliases', 'config.yaml')))
+ assert 'log_file' in config
+ assert 'aliases' in config['puppetdb']
+ assert 'role1' in config['puppetdb']['aliases']
+ assert config['puppetdb']['aliases']['role1'] == 'R:Class = Role::Role1'
+ assert 'aliases' in config['direct']
+ assert 'group1' in config['direct']['aliases']
+ assert config['direct']['aliases']['group1'] == 'host10[10-22].example.org'
+
+
+def test_config_class_empty_aliases():
+ """A CuminError is raised if one of the backend aliases is empty."""
+ with pytest.raises(cumin.CuminError, match='Empty configuration found in'):
+ cumin.Config(config=get_fixture_path(os.path.join('config',
'valid_with_empty_aliases', 'config.yaml')))
+
+
+def test_config_class_invalid_aliases():
+ """A CuminError is raised if one of the backend aliases is invalid."""
+ with pytest.raises(cumin.CuminError, match='Unable to parse configuration
file'):
+ cumin.Config(config=get_fixture_path(os.path.join('config',
'valid_with_invalid_aliases', 'config.yaml')))
def test_parse_config_ok():
"""The configuration file is properly parsed and accessible."""
- config = cumin.parse_config('doc/examples/config.yaml')
+ config = cumin.parse_config(get_fixture_path(os.path.join('config',
'valid', 'config.yaml')))
assert 'log_file' in config
@@ -29,23 +68,46 @@
def test_parse_config_invalid():
"""A CuminError is raised if the configuration cannot be parsed."""
- invalid_yaml = '\n'.join((
- 'foo:',
- ' bar: baz',
- ' - foobar',
- ))
- tmpfile, tmpfilepath = tempfile.mkstemp(suffix='config.yaml',
prefix='cumin', text=True)
- os.write(tmpfile, invalid_yaml)
-
with pytest.raises(cumin.CuminError, match='Unable to parse configuration
file'):
- cumin.parse_config(tmpfilepath)
+ cumin.parse_config(get_fixture_path(os.path.join('config', 'invalid',
'config.yaml')))
def test_parse_config_empty():
"""A CuminError is raised if the configuration is empty."""
- empty_yaml = ''
- tmpfile, tmpfilepath = tempfile.mkstemp(suffix='config.yaml',
prefix='cumin', text=True)
- os.write(tmpfile, empty_yaml)
-
with pytest.raises(cumin.CuminError, match='Empty configuration found in'):
- cumin.parse_config(tmpfilepath)
+ cumin.parse_config(get_fixture_path(os.path.join('config', 'empty',
'config.yaml')))
+
+
+def test_load_backend_aliases_missing():
+ """If no aliases file is present, load_backend_aliases() should not raise
any error."""
+ base_path = get_fixture_path(os.path.join('config', 'valid'))
+ config = {}
+ cumin.load_backend_aliases(config, base_path)
+ assert config == {}
+
+
+def test_load_backend_aliases_valid():
+ """If valid aliases files are present, load_backend_aliases() should load
them into the configuration."""
+ base_path = get_fixture_path(os.path.join('config', 'valid_with_aliases'))
+ config = {'direct': {}}
+ cumin.load_backend_aliases(config, base_path)
+ assert 'aliases' in config['puppetdb']
+ assert 'role1' in config['puppetdb']['aliases']
+ assert config['puppetdb']['aliases']['role1'] == 'R:Class = Role::Role1'
+ assert 'aliases' in config['direct']
+ assert 'group1' in config['direct']['aliases']
+ assert config['direct']['aliases']['group1'] == 'host10[10-22].example.org'
+
+
+def test_load_backend_aliases_empty():
+ """If empty aliases files are present, load_backend_aliases() should raise
CuminError."""
+ base_path = get_fixture_path(os.path.join('config',
'valid_with_empty_aliases'))
+ with pytest.raises(cumin.CuminError, match='Empty configuration found in'):
+ cumin.load_backend_aliases({}, base_path)
+
+
+def test_load_backend_aliases_invalid():
+ """If invalid aliases files are present, load_backend_aliases() should
raise CuminError."""
+ base_path = get_fixture_path(os.path.join('config',
'valid_with_invalid_aliases'))
+ with pytest.raises(cumin.CuminError, match='Unable to parse configuration
file'):
+ cumin.load_backend_aliases({}, base_path)
--
To view, visit https://gerrit.wikimedia.org/r/363747
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: Ief4c374408741af0e29d9bf07d8976b8f753d78d
Gerrit-PatchSet: 1
Gerrit-Project: operations/software/cumin
Gerrit-Branch: master
Gerrit-Owner: Volans <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
