[Cloud-init-dev] [Merge] ~chad.smith/cloud-init:bug/1840080-ubuntu-drivers-emit-latelink into cloud-init:master

2019-08-20 Thread Chad Smith
Chad Smith has proposed merging 
~chad.smith/cloud-init:bug/1840080-ubuntu-drivers-emit-latelink into 
cloud-init:master.

Commit message:
ubuntu-drivers: call db_x_loadtemplatefile to accept NVIDIA EULA

Emit a script allowing cloud-init to set linux/nvidia/latelink
debconf selection to true. This avoids having to call
debconf-set-selections and allows cloud-init to pre-confgure
linux-restricted-modules to link NVIDIA drivers to the running kernel.

Cloud-init loads this debconf template and sets the value to true in the
debconf database by sourcing debconf's /usr/share/debconf/confmodule and
uses db_x_loadtemplatefile to register cloud-init's setting for
linux/nvidia/latelink.

LP: #1840080


Requested reviews:
  cloud-init commiters (cloud-init-dev)
Related bugs:
  Bug #1840080 in cloud-init (Ubuntu): "cloud-init cc_ubuntu_drivers does not 
set up /etc/default/linux-modules-nvidia"
  https://bugs.launchpad.net/ubuntu/+source/cloud-init/+bug/1840080

For more details, see:
https://code.launchpad.net/~chad.smith/cloud-init/+git/cloud-init/+merge/371545
-- 
Your team cloud-init commiters is requested to review the proposed merge of 
~chad.smith/cloud-init:bug/1840080-ubuntu-drivers-emit-latelink into 
cloud-init:master.
diff --git a/cloudinit/config/cc_ubuntu_drivers.py b/cloudinit/config/cc_ubuntu_drivers.py
index 4da34ee..7f52987 100644
--- a/cloudinit/config/cc_ubuntu_drivers.py
+++ b/cloudinit/config/cc_ubuntu_drivers.py
@@ -9,6 +9,7 @@ from cloudinit.config.schema import (
 get_schema_doc, validate_cloudconfig_schema)
 from cloudinit import log as logging
 from cloudinit.settings import PER_INSTANCE
+from cloudinit import temp_utils
 from cloudinit import type_utils
 from cloudinit import util
 
@@ -65,6 +66,39 @@ OLD_UBUNTU_DRIVERS_STDERR_NEEDLE = (
 __doc__ = get_schema_doc(schema)  # Supplement python help()
 
 
+# debconf template to allow cloud-init pre-configure the global debconf
+# variable linux/nvidia/latelink to true, allowing linux-restricted-modules
+# to accept the NVIDIA EULA and automatically link drivers to the running
+# kernel.
+# EOL_XENIAL: can then drop this script and use python3-debconf which is only
+# available in Bionic and later. Can't use python3-debconf currently as it
+# isn't in Xenial and doesn't yet support X_LOADTEMPLATEFILE debconf command.
+
+NVIDIA_DRIVER_LATELINK_DEBCONF_TMPL = """\
+#/bin/sh
+# Allow cloud-init to trigger EULA acceptance via registering a debconf
+# template to set linux/nvidia/latelink true
+
+. /usr/share/debconf/confmodule
+
+SCRIPT=$(readlink -f "$0")
+DIRNAME=$(dirname "$SCRIPT")
+tmpfile=$(mktemp -p ${DIRNAME} -t "cloud-init-ubuntu-drivers-XX.template")
+cat > "$tmpfile" << EOF
+Template: linux/nvidia/latelink
+Type: boolean
+Default: true
+Description: Late-link NVIDIA kernel modules?
+ Enable this to link the NVIDIA kernel modules in cloud-init and
+ make them available for use.
+EOF
+echo BEFORE $tmpfile
+db_x_loadtemplatefile "$tmpfile" cloud-init
+echo AFTER $tmpfile
+rm "$tmpfile"
+"""
+
+
 def install_drivers(cfg, pkg_install_func):
 if not isinstance(cfg, dict):
 raise TypeError(
@@ -90,9 +124,10 @@ def install_drivers(cfg, pkg_install_func):
 if version_cfg:
 driver_arg += ':{}'.format(version_cfg)
 
-LOG.debug("Installing NVIDIA drivers (%s=%s, version=%s)",
+LOG.debug("Installing and activating NVIDIA drivers (%s=%s, version=%s)",
   cfgpath, nv_acc, version_cfg if version_cfg else 'latest')
 
+<<< cloudinit/config/cc_ubuntu_drivers.py
 # Setting NVIDIA latelink confirms acceptance of EULA for the package
 # linux-restricted-modules
 # Reference code defining debconf variable is here
@@ -101,6 +136,21 @@ def install_drivers(cfg, pkg_install_func):
 # nvidia.templates.in
 selections = b'linux-restricted-modules linux/nvidia/latelink boolean true'
 cc_apt_configure.debconf_set_selections(selections)
+===
+# Register and set debconf selection linux/nvidia/latelink = true
+with temp_utils.ExtendedTemporaryFile(
+suffix=".sh", needs_exe=True) as tmpf:
+try:
+tmpf.write(util.encode_text(NVIDIA_DRIVER_LATELINK_DEBCONF_TMPL))
+tmpf.flush()
+util.chmod(tmpf.name, 0o755)
+util.subp([tmpf.name])
+except Exception as e:
+util.logexc(
+LOG,
+"Failed to register NVIDIA debconf template: %s", str(e))
+raise
+>>> cloudinit/config/cc_ubuntu_drivers.py
 
 try:
 util.subp(['ubuntu-drivers', 'install', '--gpgpu', driver_arg])
diff --git a/cloudinit/config/tests/test_ubuntu_drivers.py b/cloudinit/config/tests/test_ubuntu_drivers.py
index 6a763bd..c5ee366 100644
--- a/cloudinit/config/tests/test_ubuntu_drivers.py
+++ b/cloudinit/config/tests/test_ubuntu_drivers.py
@@ -9,11 +9,20 @@ from cloudinit.config import cc_ubuntu_drivers as drivers
 from cloudinit.util import ProcessExecutionError
 
 MPATH = 

[Cloud-init-dev] [Merge] ~chad.smith/cloud-init:bug/1840080-ubuntu-drivers-emit-latelink into cloud-init:master

2019-08-15 Thread Chad Smith
Chad Smith has proposed merging 
~chad.smith/cloud-init:bug/1840080-ubuntu-drivers-emit-latelink into 
cloud-init:master.

Commit message:
ubuntu-drivers: emit latelink=true to /etc/default/ to accept nvidia eula

To accept NVIDIA EULA, cloud-init needs to emit latelink=true to the INI
file /etc/default/linux-modules-nvidia prior to installing nvidia drivers
with the ubuntu-drivers command. This will allow NVIDIA modules
prior to installing drivers enabled for linking to the running kernel.

LP: #1840080

Requested reviews:
  cloud-init commiters (cloud-init-dev)
Related bugs:
  Bug #1840080 in cloud-init (Ubuntu): "cloud-init cc_ubuntu_drivers does not 
set up /etc/default/linux-modules-nvidia"
  https://bugs.launchpad.net/ubuntu/+source/cloud-init/+bug/1840080

For more details, see:
https://code.launchpad.net/~chad.smith/cloud-init/+git/cloud-init/+merge/371369
-- 
Your team cloud-init commiters is requested to review the proposed merge of 
~chad.smith/cloud-init:bug/1840080-ubuntu-drivers-emit-latelink into 
cloud-init:master.
diff --git a/cloudinit/config/cc_ubuntu_drivers.py b/cloudinit/config/cc_ubuntu_drivers.py
index 91feb60..593b1b0 100644
--- a/cloudinit/config/cc_ubuntu_drivers.py
+++ b/cloudinit/config/cc_ubuntu_drivers.py
@@ -2,6 +2,7 @@
 
 """Ubuntu Drivers: Interact with third party drivers in Ubuntu."""
 
+import os
 from textwrap import dedent
 
 from cloudinit.config.schema import (
@@ -61,9 +62,48 @@ schema = {
 OLD_UBUNTU_DRIVERS_STDERR_NEEDLE = (
 "ubuntu-drivers: error: argument : invalid choice: 'install'")
 
+ETC_DEFAULT_FILE_NVIDIA='/etc/default/linux-modules-nvidia'
+
 __doc__ = get_schema_doc(schema)  # Supplement python help()
 
 
+def ammend_driver_defaults(config, config_file):
+"""Update INI-type config_file with config values provided.
+
+Create config_file if it doesn't exist.
+
+Other tools in  linux-restricted-modules cloud have been executed to write
+the config_file. So, preserve any pre-existing content and updating
+config keys to new values if already present.
+
+Append config key=value lines if key is not yet in
+config_file.
+
+@param config: Dict of key value pairs to write or update in config_file
+@param config_file: path to defaults file.
+"""
+if not config_file:
+config_file = ETC_DEFAULT_FILE_NVIDIA
+if os.path.exists(config_file):
+lines = util.load_file(config_file).splitlines()
+replaced_keys = set()
+for idx, line in enumerate(lines):
+for key in config.keys():
+if line.startswith('{k}='.format(k=key)):
+lines[idx] = '{k}={v}'.format(k=key, v=config[key])
+replaced_keys.update([key])
+break
+new_keys = set(config.keys()).difference(replaced_keys)
+lines.extend(
+['{k}={v}'.format(k=k, v=config[k]) for k in new_keys] + [''])
+else:
+lines = [
+'{k}={v}'.format(k=k, v=v) for (k, v) in sorted(config.items())]
+lines.insert(0, '# Written by cloud-init #cloud-config')
+lines.append('')
+util.write_file(config_file, '\n'.join(lines))
+
+
 def install_drivers(cfg, pkg_install_func):
 if not isinstance(cfg, dict):
 raise TypeError(
@@ -92,6 +132,8 @@ def install_drivers(cfg, pkg_install_func):
 LOG.debug("Installing NVIDIA drivers (%s=%s, version=%s)",
   cfgpath, nv_acc, version_cfg if version_cfg else 'latest')
 
+ammend_driver_defaults({'latelink':'true'}, ETC_DEFAULT_FILE_NVIDIA)
+
 try:
 util.subp(['ubuntu-drivers', 'install', '--gpgpu', driver_arg])
 except util.ProcessExecutionError as exc:
diff --git a/cloudinit/config/tests/test_ubuntu_drivers.py b/cloudinit/config/tests/test_ubuntu_drivers.py
index efba4ce..73f6fda 100644
--- a/cloudinit/config/tests/test_ubuntu_drivers.py
+++ b/cloudinit/config/tests/test_ubuntu_drivers.py
@@ -6,7 +6,7 @@ from cloudinit.tests.helpers import CiTestCase, skipUnlessJsonSchema, mock
 from cloudinit.config.schema import (
 SchemaValidationError, validate_cloudconfig_schema)
 from cloudinit.config import cc_ubuntu_drivers as drivers
-from cloudinit.util import ProcessExecutionError
+from cloudinit.util import ProcessExecutionError, load_file, write_file
 
 MPATH = "cloudinit.config.cc_ubuntu_drivers."
 OLD_UBUNTU_DRIVERS_ERROR_STDERR = (
@@ -14,6 +14,26 @@ OLD_UBUNTU_DRIVERS_ERROR_STDERR = (
 "(choose from 'list', 'autoinstall', 'devices', 'debug')\n")
 
 
+class TestAmmendDriverDefaults(CiTestCase):
+
+def test_write_new_file_sorted_if_absent(self):
+"""Create config_file with cloud-init header comment if absent."""
+outfile = self.tmp_path('driver-config')
+drivers.ammend_driver_defaults({'k2': 'v2', 'k1': 'v1'}, outfile)
+expected = '# Written by cloud-init #cloud-config\nk1=v1\nk2=v2\n'
+self.assertEqual(expected, load_file(outfile))
+
+def test_ammend_keys_if_present(self):
+