[MediaWiki-commits] [Gerrit] operations...puppet-compiler[master]: Add script to populate PuppetDB
jenkins-bot has submitted this change and it was merged. (
https://gerrit.wikimedia.org/r/370803 )
Change subject: Add script to populate PuppetDB
..
Add script to populate PuppetDB
Bug: T150456
Change-Id: I9bb4d2b6ebf7a5f0b20a43d47c30169e467d1c50
---
A puppet_compiler/populate_puppetdb.py
M puppet_compiler/puppet.py
M setup.py
3 files changed, 103 insertions(+), 7 deletions(-)
Approvals:
Giuseppe Lavagetto: Looks good to me, approved
jenkins-bot: Verified
diff --git a/puppet_compiler/populate_puppetdb.py
b/puppet_compiler/populate_puppetdb.py
new file mode 100644
index 000..44e2f12
--- /dev/null
+++ b/puppet_compiler/populate_puppetdb.py
@@ -0,0 +1,66 @@
+import argparse
+import logging
+import os
+import tempfile
+import shutil
+
+from puppet_compiler import nodegen, prepare, directories, puppet, utils
+
+
+parser = argparse.ArgumentParser(
+description="Puppetdb filler - this script allows to properly populate
PuppetDB with data useful for the puppet compiler"
+)
+parser.add_argument(
+'--basedir',
+default='/var/lib/catalog-differ',
+help='The base dir of the compiler installation',
+)
+parser.add_argument('--debug', action='store_true', default=False, help="Print
debug output")
+
+
+def main():
+opts = parser.parse_args()
+if opts.debug:
+lvl = logging.DEBUG
+else:
+lvl = logging.INFO
+
+logging.basicConfig(
+format='%(asctime)s %(levelname)s: %(message)s',
+level=lvl,
+datefmt='[ %Y-%m-%dT%H:%M:%S ]'
+)
+
+config = {
+'puppet_var': os.path.join(opts.basedir, 'puppet'),
+'puppet_src': os.path.join(opts.basedir, 'production'),
+'puppet_private': os.path.join(opts.basedir, 'private')
+}
+# Do the whole compilation in a dedicated directory.
+tmpdir = tempfile.mkdtemp(prefix='fill-puppetdb')
+jobid = '1'
+directories.FHS.setup(jobid, tmpdir)
+m = prepare.ManageCode(config, jobid, None)
+os.mkdir(m.base_dir, 0755)
+os.makedirs(m.prod_dir, 0755)
+m._prepare_dir(m.prod_dir)
+srcdir = os.path.join(m.prod_dir, 'src')
+with prepare.pushd(srcdir):
+m._copy_hiera(m.prod_dir, 'production')
+
+for node in nodegen.get_nodes(config):
+print "=" * 80
+print "Compiling catalog for {}".format(node)
+
+utils.refresh_yaml_date(utils.facts_file(config['puppet_var'], node))
+succ, out, err = puppet.compile_storeconfigs(node,
config['puppet_var'])
+if succ:
+print "OK"
+else:
+for line in err:
+print line
+shutil.rmtree(tmpdir)
+
+
+if __name__ == '__main__':
+main()
diff --git a/puppet_compiler/puppet.py b/puppet_compiler/puppet.py
index e33dfd5..a48931f 100644
--- a/puppet_compiler/puppet.py
+++ b/puppet_compiler/puppet.py
@@ -1,14 +1,14 @@
import os
import re
import subprocess
+
from tempfile import SpooledTemporaryFile as spoolfile
-from directories import HostFiles, FHS
+
+from puppet_compiler import _log
+from puppet_compiler.directories import HostFiles, FHS
-def compile(hostname, label, vardir, *extra_flags):
-"""
-Compile the catalog
-"""
+def compile_cmd_env(hostname, label, vardir, *extra_flags):
env = os.environ.copy()
if label == 'prod':
basedir = FHS.prod_dir
@@ -29,6 +29,14 @@
'--color=false'
]
cmd.extend(extra_flags)
+return (cmd, env)
+
+
+def compile(hostname, label, vardir, *extra_flags):
+"""
+Compile the catalog
+"""
+cmd, env = compile_cmd_env(hostname, label, vardir, *extra_flags)
hostfiles = HostFiles(hostname)
with open(hostfiles.file_for(label, 'errors'), 'w') as err:
@@ -41,3 +49,24 @@
for line in out:
if not re.match('(Info|[Nn]otice|[Ww]arning)', line):
f.write(line)
+
+
+def compile_storeconfigs(hostname, vardir):
+"""
+Specialized function to store data into puppetdb
+when compiling.
+"""
+cmd, env = compile_cmd_env(hostname, 'prod')
+out = spoolfile()
+err = spoolfile()
+success = False
+
+try:
+subprocess.check_call(cmd, stdout=out, stderr=err, env=env)
+success = True
+except subprocess.CalledProcessError as e:
+_log.exception("Compilation failed for host %s: %s", hostname, e)
+
+out.seek(0)
+err.seek(0)
+return (success, out, err)
diff --git a/setup.py b/setup.py
index c90625c..e3f4083 100755
--- a/setup.py
+++ b/setup.py
@@ -12,7 +12,7 @@
setup(
name='puppet_compiler',
-version='0.3.2',
+version='0.3.3',
description='Tools to compile puppet catalogs as a service',
author='Joe',
author_email='glavage...@wikimedia.org',
@@ -24,7 +24,8 @@
package_data={'puppet_compiler': all_files("puppet_compiler",
"templates")},
entry_points={
'console_scripts': [
-'puppet-compiler = puppet_compi
[MediaWiki-commits] [Gerrit] operations...puppet-compiler[master]: Add script to populate PuppetDB
Giuseppe Lavagetto has uploaded a new change for review. (
https://gerrit.wikimedia.org/r/370803 )
Change subject: Add script to populate PuppetDB
..
Add script to populate PuppetDB
Bug: T150456
Change-Id: I9bb4d2b6ebf7a5f0b20a43d47c30169e467d1c50
---
A puppet_compiler/populate_puppetdb.py
M puppet_compiler/puppet.py
M setup.py
3 files changed, 102 insertions(+), 7 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/operations/software/puppet-compiler
refs/changes/03/370803/1
diff --git a/puppet_compiler/populate_puppetdb.py
b/puppet_compiler/populate_puppetdb.py
new file mode 100644
index 000..83fd712
--- /dev/null
+++ b/puppet_compiler/populate_puppetdb.py
@@ -0,0 +1,65 @@
+import argparse
+import logging
+import os
+import tempfile
+import shutil
+
+from puppet_compiler import nodegen, prepare, directories, puppet, utils
+
+
+parser = argparse.ArgumentParser(
+description="Puppetdb filler - this script allows to properly populate
PuppetDB with data useful for the puppet compiler"
+)
+parser.add_argument(
+'--basedir',
+default='/var/lib/catalog-differ',
+help='The base dir of the compiler installation',
+)
+parser.add_argument('--debug', action='store_true', default=False, help="Print
debug output")
+
+
+def main():
+opts = parser.parse_args()
+if opts.debug:
+lvl = logging.DEBUG
+else:
+lvl = logging.INFO
+
+logging.basicConfig(
+format='%(asctime)s %(levelname)s: %(message)s',
+level=lvl,
+datefmt='[ %Y-%m-%dT%H:%M:%S ]'
+)
+
+config = {
+'puppet_var': os.path.join(opts.basedir, 'puppet'),
+'puppet_src': os.path.join(opts.basedir, 'production'),
+'puppet_private': os.path.join(opts.basedir, 'private')
+}
+# Do the whole compilation in a dedicated directory.
+tmpdir = tempfile.mkdtemp(prefix='fill-puppetdb')
+jobid = '1'
+directories.FHS.setup(jobid, tmpdir)
+m = prepare.ManageCode(config, jobid, None)
+os.mkdir(m.base_dir, 0755)
+os.makedirs(m.prod_dir, 0755)
+m._prepare_dir(m.prod_dir)
+srcdir = os.path.join(m.prod_dir, 'src')
+with prepare.pushd(srcdir):
+m._copy_hiera(m.prod_dir, 'production')
+
+for node in nodegen.get_nodes(config):
+print "=" * 80
+print "Compiling catalog for {}".format(node)
+
+utils.refresh_yaml_date(utils.facts_file(config['puppet_var'], node))
+succ, out, err = puppet.compile_storeconfigs(node,
config['puppet_var'])
+if succ:
+print "OK"
+else:
+for line in err:
+print line
+shutil.rmtree(tmpdir)
+
+if __name__ == '__main__':
+main()
diff --git a/puppet_compiler/puppet.py b/puppet_compiler/puppet.py
index e33dfd5..a48931f 100644
--- a/puppet_compiler/puppet.py
+++ b/puppet_compiler/puppet.py
@@ -1,14 +1,14 @@
import os
import re
import subprocess
+
from tempfile import SpooledTemporaryFile as spoolfile
-from directories import HostFiles, FHS
+
+from puppet_compiler import _log
+from puppet_compiler.directories import HostFiles, FHS
-def compile(hostname, label, vardir, *extra_flags):
-"""
-Compile the catalog
-"""
+def compile_cmd_env(hostname, label, vardir, *extra_flags):
env = os.environ.copy()
if label == 'prod':
basedir = FHS.prod_dir
@@ -29,6 +29,14 @@
'--color=false'
]
cmd.extend(extra_flags)
+return (cmd, env)
+
+
+def compile(hostname, label, vardir, *extra_flags):
+"""
+Compile the catalog
+"""
+cmd, env = compile_cmd_env(hostname, label, vardir, *extra_flags)
hostfiles = HostFiles(hostname)
with open(hostfiles.file_for(label, 'errors'), 'w') as err:
@@ -41,3 +49,24 @@
for line in out:
if not re.match('(Info|[Nn]otice|[Ww]arning)', line):
f.write(line)
+
+
+def compile_storeconfigs(hostname, vardir):
+"""
+Specialized function to store data into puppetdb
+when compiling.
+"""
+cmd, env = compile_cmd_env(hostname, 'prod')
+out = spoolfile()
+err = spoolfile()
+success = False
+
+try:
+subprocess.check_call(cmd, stdout=out, stderr=err, env=env)
+success = True
+except subprocess.CalledProcessError as e:
+_log.exception("Compilation failed for host %s: %s", hostname, e)
+
+out.seek(0)
+err.seek(0)
+return (success, out, err)
diff --git a/setup.py b/setup.py
index c90625c..e3f4083 100755
--- a/setup.py
+++ b/setup.py
@@ -12,7 +12,7 @@
setup(
name='puppet_compiler',
-version='0.3.2',
+version='0.3.3',
description='Tools to compile puppet catalogs as a service',
author='Joe',
author_email='glavage...@wikimedia.org',
@@ -24,7 +24,8 @@
package_data={'puppet_compiler': all_files("puppet_compiler",
"templates")},
entry_points={
'console_scripts': [
-'puppet-compi
