Giuseppe Lavagetto has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/370802 )

Change subject: Better support puppetdb
......................................................................

Better support puppetdb

* Copy over routes.yaml as well
* Refresh facts timestamp before compiling.

Bug: T150456
Change-Id: I2a65a5383651cd9e07fd26cb21c55e0e1e120bb2
---
M puppet_compiler/prepare.py
M puppet_compiler/tests/test_hostworker.py
A puppet_compiler/utils.py
M puppet_compiler/worker.py
4 files changed, 51 insertions(+), 5 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/operations/software/puppet-compiler 
refs/changes/02/370802/1

diff --git a/puppet_compiler/prepare.py b/puppet_compiler/prepare.py
index 5faf5a5..d7e9d24 100644
--- a/puppet_compiler/prepare.py
+++ b/puppet_compiler/prepare.py
@@ -96,13 +96,18 @@
 
         shutil.copytree(os.path.join(self.puppet_var, 'ssl'),
                         os.path.join(src, 'ssl'))
-        puppetdb_conf = os.path.join(self.puppet_src, 'puppetdb.conf')
+        # Puppetdb-related configs
+        puppetdb_conf = os.path.join(self.puppet_var, 'puppetdb.conf')
         if os.path.isfile(puppetdb_conf):
             _log.debug('Copying the puppetdb config file')
             shutil.copy(
                 puppetdb_conf,
                 os.path.join(src, 'puppetdb.conf')
             )
+        routes_conf = os.path.join(self.puppet_var, 'routes.yaml')
+        if os.path.isfile(routes_conf):
+            _log.debug('Copying the routes file')
+            shutil.copy(routes_conf, os.path.join(src, 'routes.yaml'))
 
     @staticmethod
     def _copy_hiera(dirname, realm):
diff --git a/puppet_compiler/tests/test_hostworker.py 
b/puppet_compiler/tests/test_hostworker.py
index 013e0e9..b6f9777 100644
--- a/puppet_compiler/tests/test_hostworker.py
+++ b/puppet_compiler/tests/test_hostworker.py
@@ -101,7 +101,9 @@
     def test_run_host(self):
         self.hw.facts_file = mock.Mock(return_value=False)
         self.assertEquals(self.hw.run_host(), (True, True, None))
-        self.hw.facts_file = mock.Mock(return_value=True)
+        fname = os.path.join(self.fixtures, 'puppet_var', 'yaml',
+                             'facts', 'test.eqiad.wmnet')
+        self.hw.facts_file.return_value = fname
         self.hw._compile_all = mock.Mock(return_value=0)
         self.hw._make_diff = mock.Mock(return_value=True)
         self.hw._make_output = mock.Mock(return_value=None)
@@ -112,6 +114,7 @@
         assert self.hw._make_diff.called
         assert self.hw._make_output.called
         assert self.hw._build_html.called
+
         self.hw._compile_all.return_value = 1
         self.hw._make_diff.reset_mock()
         self.assertEquals(self.hw.run_host(), (True, False, None))
diff --git a/puppet_compiler/utils.py b/puppet_compiler/utils.py
new file mode 100644
index 0000000..40b3ca4
--- /dev/null
+++ b/puppet_compiler/utils.py
@@ -0,0 +1,35 @@
+import re
+import os
+import shutil
+
+from datetime import datetime, timedelta
+
+
+def facts_file(vardir, hostname):
+    """Finds facts file for the given hostname."""
+    return os.path.join(vardir, 'yaml', 'facts',
+                        '{}.yaml'.format(hostname))
+
+
+def refresh_yaml_date(facts_file):
+    """
+    Refresh the timestamp and the expiration of the yaml facts cache
+    to avoid incurring in https://tickets.puppetlabs.com/browse/PUP-5441
+    when using puppetdb.
+    """
+    # No, we cannot read the yaml. It contains ruby data structures.
+    date_format = '%Y-%m-%d %H:%M:%S.%s +00:00'
+    print "Patching {}".format(facts_file)
+    ts_re = r'(\s+\"_timestamp\":) .*'
+    exp_re = r'(\s+expiration:) .*'
+    datetime_facts = datetime.utcnow()
+    datetime_exp = (datetime_facts + timedelta(days=1))
+    ts_sub = r'\1 {}'.format(datetime_facts.strftime(date_format))
+    exp_sub = r'\1 {}'.format(datetime_exp.strftime(date_format))
+    with open(facts_file, 'r') as fh:
+        with open(facts_file + '.tmp', 'w') as tmp:
+            for line in fh:
+                line = re.sub(ts_re, ts_sub, line)
+                line = re.sub(exp_re, exp_sub, line)
+                tmp.write(line)
+    shutil.move(facts_file + '.tmp', facts_file)
diff --git a/puppet_compiler/worker.py b/puppet_compiler/worker.py
index 654c4b2..9c695b0 100644
--- a/puppet_compiler/worker.py
+++ b/puppet_compiler/worker.py
@@ -2,7 +2,7 @@
 import shutil
 import subprocess
 
-from puppet_compiler import puppet, _log
+from puppet_compiler import puppet, _log, utils
 from puppet_compiler.filter import itos, flatten
 from puppet_compiler.differ import PuppetCatalog
 from puppet_compiler.directories import HostFiles, FHS
@@ -33,8 +33,7 @@
 
     def facts_file(self):
         """ Finds facts file for the current hostname """
-        facts_file = os.path.join(self.puppet_var, 'yaml', 'facts',
-                                  '{}.yaml'.format(self.hostname))
+        facts_file = utils.facts_file(self.puppet_var, self.hostname)
         if os.path.isfile(facts_file):
             return facts_file
         return None
@@ -47,6 +46,10 @@
             _log.error('Unable to find facts for host %s, skipping',
                        self.hostname)
             return (True, True, None)
+        else:
+            # Refresh the facts file first
+            utils.refresh_yaml_date(self.facts_file())
+
         errors = self._compile_all()
         if errors == self.E_OK:
             diff = self._make_diff()

-- 
To view, visit https://gerrit.wikimedia.org/r/370802
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: I2a65a5383651cd9e07fd26cb21c55e0e1e120bb2
Gerrit-PatchSet: 1
Gerrit-Project: operations/software/puppet-compiler
Gerrit-Branch: master
Gerrit-Owner: Giuseppe Lavagetto <glavage...@wikimedia.org>

_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits

Reply via email to