This needs lots of work, but it can successfully launch an LXC-based
instance. See the docstring for the limitations/work to be done.
---
Note that I'm not convinced LXC (as available in Debian sid, 0.6.5) is
the best way to configure linux containers. While it does a lot of
background work, there are two main issues with it:
- error handling is very weak; I had to do ip l del veth57d7 or
brctl delif xen-br0 vethfj4j3 many times
- it trips over itself easily, listing duplicate instances, etc.
- bugs and commits as recent as two months ago show that basic
functionality is stil under flux, and thus I'd estimate another 6
months to a year before it gets really stable
See also the docstring for planned enhancements for this hv.
This won't be committed before 2.1.2 is released, by the way. Still at
RFC stage or so.
Makefile.am | 1 +
lib/constants.py | 12 ++-
lib/hypervisor/__init__.py | 2 +
lib/hypervisor/hv_lxc.py | 339 ++++++++++++++++++++++++++++++++++++++++++++
4 files changed, 353 insertions(+), 1 deletions(-)
create mode 100644 lib/hypervisor/hv_lxc.py
diff --git a/Makefile.am b/Makefile.am
index 9d93c59..5f57b32 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -119,6 +119,7 @@ hypervisor_PYTHON = \
lib/hypervisor/hv_chroot.py \
lib/hypervisor/hv_fake.py \
lib/hypervisor/hv_kvm.py \
+ lib/hypervisor/hv_lxc.py \
lib/hypervisor/hv_xen.py
rapi_PYTHON = \
diff --git a/lib/constants.py b/lib/constants.py
index c6d8900..e45968f 100644
--- a/lib/constants.py
+++ b/lib/constants.py
@@ -506,7 +506,15 @@ HT_FAKE = "fake"
HT_XEN_HVM = "xen-hvm"
HT_KVM = "kvm"
HT_CHROOT = "chroot"
-HYPER_TYPES = frozenset([HT_XEN_PVM, HT_FAKE, HT_XEN_HVM, HT_KVM, HT_CHROOT])
+HT_LXC = "lxc"
+HYPER_TYPES = frozenset([
+ HT_XEN_PVM,
+ HT_FAKE,
+ HT_XEN_HVM,
+ HT_KVM,
+ HT_CHROOT,
+ HT_LXC,
+ ])
HTS_REQ_PORT = frozenset([HT_XEN_HVM, HT_KVM])
VNC_BASE_PORT = 5900
@@ -734,6 +742,8 @@ HVC_DEFAULTS = {
HT_CHROOT: {
HV_INIT_SCRIPT: "/ganeti-chroot",
},
+ HT_LXC: {
+ },
}
HVC_GLOBALS = frozenset([
diff --git a/lib/hypervisor/__init__.py b/lib/hypervisor/__init__.py
index f896219..e1b39ba 100644
--- a/lib/hypervisor/__init__.py
+++ b/lib/hypervisor/__init__.py
@@ -30,6 +30,7 @@ from ganeti.hypervisor import hv_fake
from ganeti.hypervisor import hv_xen
from ganeti.hypervisor import hv_kvm
from ganeti.hypervisor import hv_chroot
+from ganeti.hypervisor import hv_lxc
_HYPERVISOR_MAP = {
@@ -38,6 +39,7 @@ _HYPERVISOR_MAP = {
constants.HT_FAKE: hv_fake.FakeHypervisor,
constants.HT_KVM: hv_kvm.KVMHypervisor,
constants.HT_CHROOT: hv_chroot.ChrootManager,
+ constants.HT_LXC: hv_lxc.LXCHypervisor,
}
diff --git a/lib/hypervisor/hv_lxc.py b/lib/hypervisor/hv_lxc.py
new file mode 100644
index 0000000..fd0cfa8
--- /dev/null
+++ b/lib/hypervisor/hv_lxc.py
@@ -0,0 +1,339 @@
+#
+#
+
+# Copyright (C) 2010 Google Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+# 02110-1301, USA.
+
+
+"""LXC hypervisor
+
+"""
+
+import os
+import os.path
+import time
+import logging
+
+from ganeti import constants
+from ganeti import errors # pylint: disable-msg=W0611
+from ganeti import utils
+from ganeti.hypervisor import hv_base
+from ganeti.errors import HypervisorError
+
+
+class LXCHypervisor(hv_base.BaseHypervisor):
+ """LXC-based virtualization.
+
+ Since current (Spring 2010) distributions are not yet ready for
+ running under a container, the following changes must be done
+ manually:
+ - remove udev
+ - disable the kernel log component of sysklogd/rsyslog/etc.,
+ otherwise they will fail to read the log, and at least rsyslog
+ will fill the filesystem with error messages
+
+ TODO:
+ - move hardcoded parameters into hypervisor parameters, once we
+ have the container-parameter support
+ - implement memory limits, but only optionally, depending on host
+ kernel support
+
+ Problems/issues:
+ - LXC is very temperamental; in daemon mode, it succeeds or fails
+ in launching the instance silently, without any error
+ indication, and when failing it can leave network interfaces
+ around, and future successful startups will list the instance
+ twice
+ - shutdown sequence of containers leaves the init 'dead', and the
+ container effectively stopped, but LXC still believes the
+ container to be running; need to investigate using the
+ notify_on_release and release_agent feature of cgroups
+
+ """
+ _ROOT_DIR = constants.RUN_GANETI_DIR + "/lxc"
+ _DEVS = [
+ "c 1:3", # /dev/null
+ "c 1:5", # /dev/zero
+ "c 1:7", # /dev/full
+ "c 1:8", # /dev/random
+ "c 1:9", # /dev/urandom
+ "c 1:10", # /dev/aio
+ "c 5:0", # /dev/tty
+ "c 5:1", # /dev/console
+ "c 5:2", # /dev/ptmx
+ "c 136:*", # first block of Unix98 PTY slaves
+ ]
+ _DENIED_CAPABILITIES = [
+ "mac_override", # Allow MAC configuration or state changes
+ # TODO: remove sys_admin too, for safety
+ #"sys_admin", # Perform a range of system administration operations
+ "sys_boot", # Use reboot(2) and kexec_load(2)
+ "sys_module", # Load and unload kernel modules
+ "sys_time", # Set system clock, set real-time (hardware) clock
+ ]
+
+ PARAMETERS = {
+ }
+
+ def __init__(self):
+ hv_base.BaseHypervisor.__init__(self)
+ if not os.path.exists(self._ROOT_DIR):
+ os.mkdir(self._ROOT_DIR)
+ if not os.path.isdir(self._ROOT_DIR):
+ raise HypervisorError("Needed path %s is not a directory" %
+ self._ROOT_DIR)
+
+ @staticmethod
+ def _GetMountSubdirs(path):
+ """Return the list of mountpoints under a given path.
+
+ This function is Linux-specific.
+
+ """
+ #TODO(iustin): investigate and document non-linux options
+ #(e.g. via mount output)
+ data = []
+ fh = open("/proc/mounts", "r")
+ try:
+ for line in fh:
+ _, mountpoint, _ = line.split(" ", 2)
+ if (mountpoint.startswith(path) and
+ mountpoint != path):
+ data.append(mountpoint)
+ finally:
+ fh.close()
+ data.sort(key=lambda x: x.count("/"), reverse=True)
+ return data
+
+ @classmethod
+ def _InstanceDir(cls, instance_name):
+ """Return the root directory for an instance.
+
+ """
+ return utils.PathJoin(cls._ROOT_DIR, instance_name)
+
+ @classmethod
+ def _InstanceConfFile(cls, instance_name):
+ """Return the root directory for an instance.
+
+ """
+ return utils.PathJoin(cls._ROOT_DIR, instance_name + ".conf")
+
+ def ListInstances(self):
+ """Get the list of running instances.
+
+ """
+ result = utils.RunCmd(["lxc-ls"])
+ if result.failed:
+ raise errors.HypervisorError("Can't run lxc-ls: %s" % result.output)
+ return result.stdout.splitlines()
+
+ def GetInstanceInfo(self, instance_name):
+ """Get instance properties.
+
+ @type instance_name: string
+ @param instance_name: the instance name
+
+ @return: (name, id, memory, vcpus, stat, times)
+
+ """
+ # TODO: read container info from the cgroup mountpoint
+ return (instance_name, 0, 0, 0, 0, 0)
+
+ def GetAllInstancesInfo(self):
+ """Get properties of all instances.
+
+ @return: [(name, id, memory, vcpus, stat, times),...]
+
+ """
+ data = []
+ for name in self.ListInstances():
+ data.append((name, 0, 0, 0, 0, 0))
+ return data
+
+ def _CreateConfigFile(self, instance, root_dir):
+ """Create an lxc.conf file for an instance"""
+ out = []
+ # hostname
+ out.append("lxc.utsname = %s" % instance.name)
+
+ # separate pseudo-TTY instances
+ out.append("lxc.pts = 255")
+ # standard TTYs/console
+ out.append("lxc.tty = 6")
+
+ # root FS
+ out.append("lxc.rootfs = %s" % root_dir)
+
+ # TODO: additional mounts, if we disable CAP_SYS_ADMIN
+
+ # Device control
+ # deny direct device access
+ out.append("lxc.cgroup.devices.deny = a")
+ for devinfo in self._DEVS:
+ out.append("lxc.cgroup.devices.allow = %s rw" % devinfo)
+
+ # Networking
+ for idx, nic in enumerate(instance.nics):
+ out.append("# NIC %d" % idx)
+ mode = nic.nicparams[constants.NIC_MODE]
+ link = nic.nicparams[constants.NIC_LINK]
+ if mode == constants.NIC_MODE_BRIDGED:
+ out.append("lxc.network.type = veth")
+ out.append("lxc.network.link = %s" % link)
+ else:
+ raise errors.HypervisorError("LXC hypervisor supports only"
+ " bridged mode (NIC %d has mode %s)" %
+ (idx, mode))
+ out.append("lxc.network.hwaddr = %s" % nic.mac)
+ out.append("lxc.network.flags = up")
+
+ # Capabilities
+ for cap in self._DENIED_CAPABILITIES:
+ out.append("lxc.cap.drop = %s" % cap)
+
+ return "\n".join(out) + "\n"
+
+ def StartInstance(self, instance, block_devices):
+ """Start an instance.
+
+ For LCX, we try to mount the block device and execute 'lxc-start
+ start' (we use volatile containers).
+
+ """
+ root_dir = self._InstanceDir(instance.name)
+ if not os.path.exists(root_dir):
+ try:
+ os.mkdir(root_dir)
+ except IOError, err:
+ raise HypervisorError("Failed to start instance %s: %s" %
+ (instance.name, err))
+ if not os.path.isdir(root_dir):
+ raise HypervisorError("Needed path %s is not a directory" % root_dir)
+
+ conf_file = self._InstanceConfFile(instance.name)
+ utils.WriteFile(conf_file, data=self._CreateConfigFile(instance, root_dir))
+
+ if not os.path.ismount(root_dir):
+ if not block_devices:
+ raise HypervisorError("LXC needs at least one disk")
+
+ sda_dev_path = block_devices[0][1]
+ result = utils.RunCmd(["mount", sda_dev_path, root_dir])
+ if result.failed:
+ raise HypervisorError("Can't mount the chroot dir: %s" % result.output)
+ result = utils.RunCmd(["lxc-start", "-n", instance.name,
+ "-o", "/tmp/lxc.log", "-l", "DEBUG",
+ "-f", conf_file, "-d"])
+ if result.failed:
+ raise HypervisorError("Can't run the lxc start script: %s" %
+ result.output)
+
+ def StopInstance(self, instance, force=False, retry=False, name=None):
+ """Stop an instance.
+
+ This method has complicated cleanup tests, as we must:
+ - try to kill all leftover processes
+ - try to unmount any additional sub-mountpoints
+ - finally unmount the instance dir
+
+ """
+ if name is None:
+ name = instance.name
+
+ root_dir = self._InstanceDir(name)
+ if not os.path.exists(root_dir):
+ return
+
+ if name in self.ListInstances():
+ # Signal init to shutdown; this is a hack
+ if not retry and not force:
+ result = utils.RunCmd(["chroot", root_dir, "poweroff"])
+ if result.failed:
+ raise HypervisorError("Can't run the lxc stop script: %s" %
+ result.output)
+ time.sleep(2)
+ utils.RunCmd(["lxc-stop", "-n", name])
+
+ for mpath in self._GetMountSubdirs(root_dir):
+ utils.RunCmd(["umount", mpath])
+
+ result = utils.RunCmd(["umount", root_dir])
+ if result.failed and force:
+ msg = ("Processes still alive in the chroot: %s" %
+ utils.RunCmd("fuser -vm %s" % root_dir).output)
+ logging.error(msg)
+ raise HypervisorError("Can't umount the chroot dir: %s (%s)" %
+ (result.output, msg))
+
+ def RebootInstance(self, instance):
+ """Reboot an instance.
+
+ This is not (yet) implemented for lxc.
+
+ """
+ raise HypervisorError("The LXC hypervisor doesn't implement the"
+ " reboot functionality")
+
+ def GetNodeInfo(self):
+ """Return information about the node.
+
+ This is just a wrapper over the base GetLinuxNodeInfo method.
+
+ @return: a dict with the following keys (values in MiB):
+ - memory_total: the total memory size on the node
+ - memory_free: the available memory on the node for instances
+ - memory_dom0: the memory used by the node itself, if available
+
+ """
+ return self.GetLinuxNodeInfo()
+
+ @classmethod
+ def GetShellCommandForConsole(cls, instance, hvparams, beparams):
+ """Return a command for connecting to the console of an instance.
+
+ """
+ return "lxc-console -n %s" % instance.name
+
+ def Verify(self):
+ """Verify the hypervisor.
+
+ For the chroot manager, it just checks the existence of the base dir.
+
+ """
+ if not os.path.exists(self._ROOT_DIR):
+ return "The required directory '%s' does not exist." % self._ROOT_DIR
+
+ @classmethod
+ def PowercycleNode(cls):
+ """LXC powercycle, just a wrapper over Linux powercycle.
+
+ """
+ cls.LinuxPowercycle()
+
+ def MigrateInstance(self, instance, target, live):
+ """Migrate an instance.
+
+ @type instance: L{objects.Instance}
+ @param instance: the instance to be migrated
+ @type target: string
+ @param target: hostname (usually ip) of the target node
+ @type live: boolean
+ @param live: whether to do a live or non-live migration
+
+ """
+ raise HypervisorError("Migration not supported by the LXC hypervisor")
--
1.7.0.4
--
Subscription settings:
http://groups.google.com/group/ganeti-devel/subscribe?hl=en
