Repository: aurora Updated Branches: refs/heads/master b912e1726 -> a071af345
Isolate the executor's filesystem from the task's. This changes the approach to launching tasks with filesystem images in the unified containerizer. Instead of adding an `Image` to the `MesosContainer`, we instead add the task filesystem as a `Volume` with an associated image. This image is mounted in the mesos directory under the `taskfs` path. The executor, on start up does the following: 1. Creates user/group under the taskfs root. 2. Bind mounts the mesos directory under the taskfs root. 2. Uses mesos-containerizer's launch subcommand to pivot into the task fs and execute each process. Reviewed at https://reviews.apache.org/r/47853/ Project: http://git-wip-us.apache.org/repos/asf/aurora/repo Commit: http://git-wip-us.apache.org/repos/asf/aurora/commit/a071af34 Tree: http://git-wip-us.apache.org/repos/asf/aurora/tree/a071af34 Diff: http://git-wip-us.apache.org/repos/asf/aurora/diff/a071af34 Branch: refs/heads/master Commit: a071af3450966ecc13994fb3111f5e77635c9881 Parents: b912e17 Author: Joshua Cohen <jco...@apache.org> Authored: Wed Aug 3 11:16:44 2016 -0500 Committer: Joshua Cohen <jco...@apache.org> Committed: Wed Aug 3 11:16:44 2016 -0500 ---------------------------------------------------------------------- .../thrift/org/apache/aurora/gen/api.thrift | 4 + examples/vagrant/upstart/aurora-scheduler.conf | 4 +- .../scheduler/mesos/MesosTaskFactory.java | 11 +- .../executor/bin/thermos_executor_main.py | 14 ++- .../apache/aurora/executor/common/sandbox.py | 125 ++++++++++++++++--- .../aurora/executor/thermos_task_runner.py | 21 +++- src/main/python/apache/thermos/core/BUILD | 1 + src/main/python/apache/thermos/core/process.py | 73 +++++++---- src/main/python/apache/thermos/core/runner.py | 8 +- .../apache/thermos/runner/thermos_runner.py | 13 +- .../mesos/MesosTaskFactoryImplTest.java | 23 ++-- .../aurora/executor/common/test_sandbox.py | 61 ++++++++- .../python/apache/thermos/core/test_process.py | 39 +++++- src/test/sh/org/apache/aurora/e2e/Dockerfile | 21 ---- .../sh/org/apache/aurora/e2e/Dockerfile.netcat | 18 +++ .../sh/org/apache/aurora/e2e/Dockerfile.python | 17 +++ .../apache/aurora/e2e/http/http_example.aurora | 20 ++- .../http/http_example_bad_healthcheck.aurora | 16 ++- .../aurora/e2e/http/http_example_updated.aurora | 16 ++- src/test/sh/org/apache/aurora/e2e/run-server.sh | 7 ++ .../sh/org/apache/aurora/e2e/test_end_to_end.sh | 20 +-- 21 files changed, 417 insertions(+), 115 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/aurora/blob/a071af34/api/src/main/thrift/org/apache/aurora/gen/api.thrift ---------------------------------------------------------------------- diff --git a/api/src/main/thrift/org/apache/aurora/gen/api.thrift b/api/src/main/thrift/org/apache/aurora/gen/api.thrift index 1d66208..b799cce 100644 --- a/api/src/main/thrift/org/apache/aurora/gen/api.thrift +++ b/api/src/main/thrift/org/apache/aurora/gen/api.thrift @@ -1201,3 +1201,7 @@ service AuroraAdmin extends AuroraSchedulerManager { // The name of the header that should be sent to bypass leader redirection in the Scheduler. const string BYPASS_LEADER_REDIRECT_HEADER_NAME = 'Bypass-Leader-Redirect' + +// The path under which a task's filesystem should be mounted when using images and the Mesos +// unified containerizer. +const string TASK_FILESYSTEM_MOUNT_POINT = 'taskfs' http://git-wip-us.apache.org/repos/asf/aurora/blob/a071af34/examples/vagrant/upstart/aurora-scheduler.conf ---------------------------------------------------------------------- diff --git a/examples/vagrant/upstart/aurora-scheduler.conf b/examples/vagrant/upstart/aurora-scheduler.conf index 954ddb4..dd60981 100644 --- a/examples/vagrant/upstart/aurora-scheduler.conf +++ b/examples/vagrant/upstart/aurora-scheduler.conf @@ -41,8 +41,8 @@ exec bin/aurora-scheduler \ -native_log_file_path=/var/db/aurora \ -backup_dir=/var/lib/aurora/backups \ -thermos_executor_path=$DIST_DIR/thermos_executor.pex \ - -global_container_mounts=/home/vagrant/aurora/examples/vagrant/config:/home/vagrant/aurora/examples/vagrant/config:ro,/etc/passwd:/etc/passwd:ro,/etc/group:/etc/group:ro \ - -thermos_executor_flags="--announcer-ensemble localhost:2181 --announcer-zookeeper-auth-config /home/vagrant/aurora/examples/vagrant/config/announcer-auth.json" \ + -global_container_mounts=/home/vagrant/aurora/examples/vagrant/config:/home/vagrant/aurora/examples/vagrant/config:ro \ + -thermos_executor_flags="--announcer-ensemble localhost:2181 --announcer-zookeeper-auth-config /home/vagrant/aurora/examples/vagrant/config/announcer-auth.json --mesos-containerizer-path=/usr/libexec/mesos/mesos-containerizer" \ -allowed_container_types=MESOS,DOCKER \ -http_authentication_mechanism=BASIC \ -use_beta_db_task_store=true \ http://git-wip-us.apache.org/repos/asf/aurora/blob/a071af34/src/main/java/org/apache/aurora/scheduler/mesos/MesosTaskFactory.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/aurora/scheduler/mesos/MesosTaskFactory.java b/src/main/java/org/apache/aurora/scheduler/mesos/MesosTaskFactory.java index f9b1c7c..0b41dba 100644 --- a/src/main/java/org/apache/aurora/scheduler/mesos/MesosTaskFactory.java +++ b/src/main/java/org/apache/aurora/scheduler/mesos/MesosTaskFactory.java @@ -64,6 +64,8 @@ import org.slf4j.LoggerFactory; import static java.util.Objects.requireNonNull; +import static org.apache.aurora.gen.apiConstants.TASK_FILESYSTEM_MOUNT_POINT; + /** * A factory to create mesos task objects. */ @@ -234,12 +236,17 @@ public interface MesosTaskFactory { ContainerInfo.MesosInfo.Builder mesosContainerBuilder = ContainerInfo.MesosInfo.newBuilder(); - mesosContainerBuilder.setImage(imageBuilder); + Protos.Volume volume = Protos.Volume.newBuilder() + .setImage(imageBuilder) + .setContainerPath(TASK_FILESYSTEM_MOUNT_POINT) + .setMode(Protos.Volume.Mode.RO) + .build(); return Optional.of(ContainerInfo.newBuilder() .setType(ContainerInfo.Type.MESOS) .setMesos(mesosContainerBuilder) - .addAllVolumes(executorSettings.getExecutorConfig().getVolumeMounts())); + .addAllVolumes(executorSettings.getExecutorConfig().getVolumeMounts()) + .addVolumes(volume)); } return Optional.absent(); http://git-wip-us.apache.org/repos/asf/aurora/blob/a071af34/src/main/python/apache/aurora/executor/bin/thermos_executor_main.py ---------------------------------------------------------------------- diff --git a/src/main/python/apache/aurora/executor/bin/thermos_executor_main.py b/src/main/python/apache/aurora/executor/bin/thermos_executor_main.py index c515931..65a495d 100644 --- a/src/main/python/apache/aurora/executor/bin/thermos_executor_main.py +++ b/src/main/python/apache/aurora/executor/bin/thermos_executor_main.py @@ -95,6 +95,14 @@ app.add_option( help='Path to ZooKeeper authentication to use for announcer nodes.') app.add_option( + '--mesos-containerizer-path', + dest='mesos_containerizer_path', + type=str, + help='The path to the mesos-containerizer executable that will be used to isolate the task''s ' + 'filesystem when using a filesystem image. Note: this path should match the value of the ' + 'Mesos Agent''s -launcher_dir flag.') + +app.add_option( '--execute-as-user', dest='execute_as_user', type=str, @@ -207,7 +215,8 @@ def initialize(options): process_logger_mode=options.runner_logger_mode, rotate_log_size_mb=options.runner_rotate_log_size_mb, rotate_log_backups=options.runner_rotate_log_backups, - preserve_env=options.preserve_env + preserve_env=options.preserve_env, + mesos_containerizer_path=options.mesos_containerizer_path ) thermos_runner_provider.set_role(None) @@ -225,7 +234,8 @@ def initialize(options): process_logger_mode=options.runner_logger_mode, rotate_log_size_mb=options.runner_rotate_log_size_mb, rotate_log_backups=options.runner_rotate_log_backups, - preserve_env=options.preserve_env + preserve_env=options.preserve_env, + mesos_containerizer_path=options.mesos_containerizer_path ) thermos_executor = AuroraExecutor( http://git-wip-us.apache.org/repos/asf/aurora/blob/a071af34/src/main/python/apache/aurora/executor/common/sandbox.py ---------------------------------------------------------------------- diff --git a/src/main/python/apache/aurora/executor/common/sandbox.py b/src/main/python/apache/aurora/executor/common/sandbox.py index 6d8b7f5..5f091af 100644 --- a/src/main/python/apache/aurora/executor/common/sandbox.py +++ b/src/main/python/apache/aurora/executor/common/sandbox.py @@ -16,12 +16,15 @@ import getpass import grp import os import pwd +import subprocess from abc import abstractmethod, abstractproperty from twitter.common import log from twitter.common.dirutil import safe_mkdir, safe_rmtree from twitter.common.lang import Interface +from gen.apache.aurora.api.constants import TASK_FILESYSTEM_MOUNT_POINT + class SandboxInterface(Interface): class Error(Exception): pass @@ -36,6 +39,10 @@ class SandboxInterface(Interface): def chrooted(self): """Returns whether or not the sandbox is a chroot.""" + @abstractproperty + def is_filesystem_image(self): + """Returns whether or not the task is using a filesystem image.""" + @abstractmethod def exists(self): """Returns true if the sandbox appears to exist.""" @@ -64,18 +71,21 @@ class DefaultSandboxProvider(SandboxProvider): def from_assigned_task(self, assigned_task): container = assigned_task.task.container if container.docker: - return FileSystemImageSandbox(self.SANDBOX_NAME) + return DockerDirectorySandbox(self.SANDBOX_NAME) elif container.mesos and container.mesos.image: return FileSystemImageSandbox(self.SANDBOX_NAME, self._get_sandbox_user(assigned_task)) else: return DirectorySandbox( - os.path.abspath(self.SANDBOX_NAME), - self._get_sandbox_user(assigned_task)) + os.path.abspath(self.SANDBOX_NAME), + self._get_sandbox_user(assigned_task)) class DirectorySandbox(SandboxInterface): """ Basic sandbox implementation using a directory on the filesystem """ + MESOS_DIRECTORY_ENV_VARIABLE = 'MESOS_DIRECTORY' + MESOS_SANDBOX_ENV_VARIABLE = 'MESOS_SANDBOX' + def __init__(self, root, user=getpass.getuser()): self._root = root self._user = user @@ -88,9 +98,23 @@ class DirectorySandbox(SandboxInterface): def chrooted(self): return False + @property + def is_filesystem_image(self): + return False + def exists(self): return os.path.exists(self.root) + def get_user_and_group(self): + try: + pwent = pwd.getpwnam(self._user) + grent = grp.getgrgid(pwent.pw_gid) + + return (pwent, grent) + except KeyError: + raise self.CreationError( + 'Could not create sandbox because user does not exist: %s' % self._user) + def create(self): log.debug('DirectorySandbox: mkdir %s' % self.root) @@ -100,12 +124,7 @@ class DirectorySandbox(SandboxInterface): raise self.CreationError('Failed to create the sandbox: %s' % e) if self._user: - try: - pwent = pwd.getpwnam(self._user) - grent = grp.getgrgid(pwent.pw_gid) - except KeyError: - raise self.CreationError( - 'Could not create sandbox because user does not exist: %s' % self._user) + pwent, grent = self.get_user_and_group() try: log.debug('DirectorySandbox: chown %s:%s %s' % (self._user, grent.gr_name, self.root)) @@ -122,19 +141,13 @@ class DirectorySandbox(SandboxInterface): raise self.DeletionError('Failed to destroy sandbox: %s' % e) -class FileSystemImageSandbox(DirectorySandbox): - """ - A sandbox implementation that configures the sandbox correctly for tasks provisioned from a - filesystem image. - """ - - MESOS_DIRECTORY_ENV_VARIABLE = 'MESOS_DIRECTORY' - MESOS_SANDBOX_ENV_VARIABLE = 'MESOS_SANDBOX' +class DockerDirectorySandbox(DirectorySandbox): + """ A sandbox implementation that configures the sandbox correctly for docker containers. """ - def __init__(self, sandbox_name, user=None): + def __init__(self, sandbox_name): self._mesos_host_sandbox = os.environ[self.MESOS_DIRECTORY_ENV_VARIABLE] self._root = os.path.join(self._mesos_host_sandbox, sandbox_name) - super(FileSystemImageSandbox, self).__init__(self._root, user=user) + super(DockerDirectorySandbox, self).__init__(self._root, user=None) def _create_symlinks(self): # This sets up the container to have a similar directory structure to the host. @@ -153,4 +166,78 @@ class FileSystemImageSandbox(DirectorySandbox): def create(self): self._create_symlinks() + super(DockerDirectorySandbox, self).create() + + +class FileSystemImageSandbox(DirectorySandbox): + """ + A sandbox implementation that configures the sandbox correctly for tasks provisioned from a + filesystem image. + """ + + # returncode from a `useradd` or `groupadd` call indicating that the uid/gid already exists. + _USER_OR_GROUP_ID_EXISTS = 4 + + def __init__(self, root, user=None): + self._task_fs_root = os.path.join( + os.environ[self.MESOS_DIRECTORY_ENV_VARIABLE], + TASK_FILESYSTEM_MOUNT_POINT) + super(FileSystemImageSandbox, self).__init__(root, user=user) + + def _create_user_and_group_in_taskfs(self): + if self._user: + pwent, grent = self.get_user_and_group() + + try: + subprocess.check_call( + ['groupadd', '-R', self._task_fs_root, '-g', '%s' % grent.gr_gid, grent.gr_name]) + except subprocess.CalledProcessError as e: + # If the failure was due to the group existing, we're ok to continue, otherwise bail out. + if e.returncode == self._USER_OR_GROUP_ID_EXISTS: + log.info( + 'Group %s(%s) already exists in the task''s filesystem, no need to create.' % ( + grent.gr_name, grent.gr_gid)) + else: + raise self.CreationError('Failed to create group in sandbox for task image: %s' % e) + + try: + subprocess.check_call([ + 'useradd', + '-R', + self._task_fs_root, + '-u', + '%s' % pwent.pw_uid, + '-g', '%s' % pwent.pw_gid, + pwent.pw_name]) + except subprocess.CalledProcessError as e: + # If the failure was due to the user existing, we're ok to continue, otherwise bail out. + if e.returncode == self._USER_OR_GROUP_ID_EXISTS: + log.info( + 'User %s (%s) already exists in the task''s filesystem, no need to create.' % ( + self._user, pwent.pw_uid)) + else: + raise self.CreationError('Failed to create user in sandbox for task image: %s' % e) + + def _mount_mesos_directory_into_taskfs(self): + mesos_directory = os.environ[self.MESOS_DIRECTORY_ENV_VARIABLE] + mount_path = os.path.join(self._task_fs_root, mesos_directory[1:]) + + log.debug('Mounting mesos directory (%s) into task filesystem at %s' % ( + mesos_directory, + mount_path)) + + safe_mkdir(mount_path) + subprocess.check_call([ + 'mount', + '--bind', + mesos_directory, + mount_path]) + + @property + def is_filesystem_image(self): + return True + + def create(self): + self._create_user_and_group_in_taskfs() + self._mount_mesos_directory_into_taskfs() super(FileSystemImageSandbox, self).create() http://git-wip-us.apache.org/repos/asf/aurora/blob/a071af34/src/main/python/apache/aurora/executor/thermos_task_runner.py ---------------------------------------------------------------------- diff --git a/src/main/python/apache/aurora/executor/thermos_task_runner.py b/src/main/python/apache/aurora/executor/thermos_task_runner.py index 3896e38..1d713ca 100644 --- a/src/main/python/apache/aurora/executor/thermos_task_runner.py +++ b/src/main/python/apache/aurora/executor/thermos_task_runner.py @@ -77,7 +77,8 @@ class ThermosTaskRunner(TaskRunner): process_logger_mode=None, rotate_log_size_mb=None, rotate_log_backups=None, - preserve_env=False): + preserve_env=False, + mesos_containerizer_path=None): """ runner_pex location of the thermos_runner pex that this task runner should use task_id task_id assigned by scheduler @@ -89,6 +90,8 @@ class ThermosTaskRunner(TaskRunner): artifact_dir scratch space for the thermos runner (basically cwd of thermos.pex) clock clock preserve_env + mesos_containerizer_path path to the mesos-containerizer executable used to isolate a task's + filesystem. """ self._runner_pex = runner_pex self._task_id = task_id @@ -98,6 +101,7 @@ class ThermosTaskRunner(TaskRunner): self._status = None self._ports = portmap self._root = sandbox.root + self._sandbox = sandbox self._checkpoint_root = checkpoint_root self._enable_chroot = sandbox.chrooted self._preserve_env = preserve_env @@ -109,6 +113,7 @@ class ThermosTaskRunner(TaskRunner): self._process_logger_mode = process_logger_mode self._rotate_log_size_mb = rotate_log_size_mb self._rotate_log_backups = rotate_log_backups + self._mesos_containerizer_path = mesos_containerizer_path # wait events self._dead = threading.Event() @@ -260,6 +265,9 @@ class ThermosTaskRunner(TaskRunner): cmdline_args.extend(['--enable_chroot']) if self._preserve_env: cmdline_args.extend(['--preserve_env']) + if self._sandbox.is_filesystem_image: + cmdline_args.extend( + ['--mesos_containerizer_path=%s' % self._mesos_containerizer_path]) for name, port in self._ports.items(): cmdline_args.extend(['--port=%s:%s' % (name, port)]) return cmdline_args @@ -365,7 +373,8 @@ class DefaultThermosTaskRunnerProvider(TaskRunnerProvider): process_logger_destination=None, process_logger_mode=None, rotate_log_size_mb=None, - rotate_log_backups=None): + rotate_log_backups=None, + mesos_containerizer_path=None): self._artifact_dir = artifact_dir or safe_mkdtemp() self._checkpoint_root = checkpoint_root self._preserve_env = preserve_env @@ -379,11 +388,16 @@ class DefaultThermosTaskRunnerProvider(TaskRunnerProvider): self._process_logger_mode = process_logger_mode self._rotate_log_size_mb = rotate_log_size_mb self._rotate_log_backups = rotate_log_backups + self._mesos_containerizer_path = mesos_containerizer_path def _get_role(self, assigned_task): return None if assigned_task.task.container.docker else assigned_task.task.job.role def from_assigned_task(self, assigned_task, sandbox): + if sandbox.is_filesystem_image and self._mesos_containerizer_path is None: + raise TaskError('Cannot launch task using a filesystem image: no mesos_containerizer_path ' + 'was set.') + task_id = assigned_task.taskId role = self._get_role(assigned_task) try: @@ -412,7 +426,8 @@ class DefaultThermosTaskRunnerProvider(TaskRunnerProvider): process_logger_mode=self._process_logger_mode, rotate_log_size_mb=self._rotate_log_size_mb, rotate_log_backups=self._rotate_log_backups, - preserve_env=self._preserve_env) + preserve_env=self._preserve_env, + mesos_containerizer_path=self._mesos_containerizer_path) return HttpLifecycleManager.wrap(runner, mesos_task, mesos_ports) http://git-wip-us.apache.org/repos/asf/aurora/blob/a071af34/src/main/python/apache/thermos/core/BUILD ---------------------------------------------------------------------- diff --git a/src/main/python/apache/thermos/core/BUILD b/src/main/python/apache/thermos/core/BUILD index 1094664..82448ce 100644 --- a/src/main/python/apache/thermos/core/BUILD +++ b/src/main/python/apache/thermos/core/BUILD @@ -25,6 +25,7 @@ python_library( '3rdparty/python:twitter.common.log', '3rdparty/python:twitter.common.quantity', '3rdparty/python:twitter.common.recordio', + 'api/src/main/thrift/org/apache/aurora/gen', 'api/src/main/thrift/org/apache/thermos', 'src/main/python/apache/thermos/common', 'src/main/python/apache/thermos/config', http://git-wip-us.apache.org/repos/asf/aurora/blob/a071af34/src/main/python/apache/thermos/core/process.py ---------------------------------------------------------------------- diff --git a/src/main/python/apache/thermos/core/process.py b/src/main/python/apache/thermos/core/process.py index 1791b5f..a296fa7 100644 --- a/src/main/python/apache/thermos/core/process.py +++ b/src/main/python/apache/thermos/core/process.py @@ -38,6 +38,7 @@ from twitter.common.lang import Interface from twitter.common.quantity import Amount, Data, Time from twitter.common.recordio import ThriftRecordReader, ThriftRecordWriter +from gen.apache.aurora.api.constants import TASK_FILESYSTEM_MOUNT_POINT from gen.apache.thermos.ttypes import ProcessState, ProcessStatus, RunnerCkpt @@ -96,7 +97,7 @@ class ProcessBase(object): def __init__(self, name, cmdline, sequence, pathspec, sandbox_dir, user=None, platform=None, logger_destination=LoggerDestination.FILE, logger_mode=LoggerMode.STANDARD, - rotate_log_size=None, rotate_log_backups=None): + rotate_log_size=None, rotate_log_backups=None, mesos_containerizer_path=None): """ required: name = name of the process @@ -114,6 +115,8 @@ class ProcessBase(object): logger_mode = The type of logger to use for the process. rotate_log_size = The maximum size of the rotated stdout/stderr logs. rotate_log_backups = The maximum number of rotated stdout/stderr log backups. + mesos_containerizer_path = The path to the mesos-containerizer binary to be used for task + filesystem isolation. """ self._name = name self._cmdline = cmdline @@ -134,6 +137,7 @@ class ProcessBase(object): self._logger_mode = logger_mode self._rotate_log_size = rotate_log_size self._rotate_log_backups = rotate_log_backups + self._mesos_containerizer_path = mesos_containerizer_path if not LoggerDestination.is_valid(self._logger_destination): raise ValueError("Logger destination %s is invalid." % self._logger_destination) @@ -182,6 +186,16 @@ class ProcessBase(object): coordinator_pid=self._pid) def cmdline(self): + if self._mesos_containerizer_path is not None: + return ('%s launch ' + '--unshare_namespace_mnt ' + '--rootfs=%s ' + '--user=%s ' + '--command=\'{"shell":true,"value":"%s"}\'' % ( + self._mesos_containerizer_path, + os.path.join(os.environ['MESOS_DIRECTORY'], TASK_FILESYSTEM_MOUNT_POINT), + self._user, + self._cmdline.replace('"', '\\"'))) return self._cmdline def name(self): @@ -344,6 +358,7 @@ class Process(ProcessBase): self._use_chroot = bool(kw.pop('chroot', False)) self._rc = None self._preserve_env = bool(kw.pop('preserve_env', False)) + kw['platform'] = RealPlatform(fork=fork) ProcessBase.__init__(self, *args, **kw) if self._use_chroot and self._sandbox is None: @@ -377,15 +392,28 @@ class Process(ProcessBase): os.setsid() if self._use_chroot: self._chroot() - self._setuid() + + # If the mesos containerizer path is set, then this process will be launched from within an + # isolated filesystem image by the mesos-containerizer executable. This executable needs to be + # run as root so that it can properly set up the filesystem as such we'll skip calling setuid at + # this point. We'll instead setuid after the process has been forked (mesos-containerizer itself + # ensures the forked process is run as the correct user). + taskfs_isolated = self._mesos_containerizer_path is not None + if not taskfs_isolated: + self._setuid() # start process start_time = self._platform.clock().time() if not self._sandbox: - sandbox = os.getcwd() + cwd = sandbox = os.getcwd() else: - sandbox = self._sandbox if not self._use_chroot else '/' + if self._use_chroot or taskfs_isolated: + sandbox = '/' + cwd = self._sandbox if taskfs_isolated else sandbox + else: + cwd = sandbox = self._sandbox + homedir = sandbox thermos_profile = os.path.join(sandbox, self.RCFILE) @@ -395,7 +423,7 @@ class Process(ProcessBase): env = {} env.update({ - 'HOME': homedir if self._use_chroot else sandbox, + 'HOME': homedir, 'LOGNAME': username, 'USER': username, 'PATH': os.environ['PATH'] @@ -403,35 +431,34 @@ class Process(ProcessBase): if os.path.exists(thermos_profile): env.update(BASH_ENV=thermos_profile) - subprocess_args = { 'args': ["/bin/bash", "-c", self.cmdline()], 'close_fds': self.FD_CLOEXEC, - 'cwd': sandbox, + 'cwd': cwd, 'env': env, 'pathspec': self._pathspec } - log_destination_resolver = LogDestinationResolver(self._pathspec, - destination=self._logger_destination, - mode=self._logger_mode, - rotate_log_size=self._rotate_log_size, - rotate_log_backups=self._rotate_log_backups) + log_destination_resolver = LogDestinationResolver( + self._pathspec, + destination=self._logger_destination, + mode=self._logger_mode, + rotate_log_size=self._rotate_log_size, + rotate_log_backups=self._rotate_log_backups) stdout, stderr, handlers_are_files = log_destination_resolver.get_handlers() if handlers_are_files: - executor = SubprocessExecutor(stdout=stdout, - stderr=stderr, - **subprocess_args) + executor = SubprocessExecutor(stdout=stdout, stderr=stderr, **subprocess_args) else: - executor = PipedSubprocessExecutor(stdout=stdout, - stderr=stderr, - **subprocess_args) + executor = PipedSubprocessExecutor(stdout=stdout, stderr=stderr, **subprocess_args) pid = executor.start() - self._write_process_update(state=ProcessState.RUNNING, - pid=pid, - start_time=start_time) + # Now that we've forked the process, if the task's filesystem is isolated it's now safe to + # setuid. + if taskfs_isolated: + self._setuid() + + self._write_process_update(state=ProcessState.RUNNING, pid=pid, start_time=start_time) rc = executor.wait() @@ -443,9 +470,7 @@ class Process(ProcessBase): else: state = ProcessState.FAILED - self._write_process_update(state=state, - return_code=rc, - stop_time=self._platform.clock().time()) + self._write_process_update(state=state, return_code=rc, stop_time=self._platform.clock().time()) self._rc = rc def finish(self): http://git-wip-us.apache.org/repos/asf/aurora/blob/a071af34/src/main/python/apache/thermos/core/runner.py ---------------------------------------------------------------------- diff --git a/src/main/python/apache/thermos/core/runner.py b/src/main/python/apache/thermos/core/runner.py index fe971ed..dcfc190 100644 --- a/src/main/python/apache/thermos/core/runner.py +++ b/src/main/python/apache/thermos/core/runner.py @@ -421,7 +421,7 @@ class TaskRunner(object): universal_handler=None, planner_class=TaskPlanner, hostname=None, process_logger_destination=None, process_logger_mode=None, rotate_log_size_mb=None, rotate_log_backups=None, - preserve_env=False): + preserve_env=False, mesos_containerizer_path=None): """ required: task (config.Task) = the task to run @@ -449,6 +449,8 @@ class TaskRunner(object): rotate_log_backups (integer) = The maximum number of rotated stdout/stderr log backups. preserve_env (boolean) = whether or not env variables for the runner should be in the env for the task being run + mesos_containerizer_path = the path to the mesos-containerizer executable that will be used + to isolate the task's filesystem (if using a filesystem image). """ if not issubclass(planner_class, TaskPlanner): raise TypeError('planner_class must be a TaskPlanner.') @@ -511,6 +513,7 @@ class TaskRunner(object): self._watcher = ProcessMuxer(self._pathspec) self._state = RunnerState(processes={}) self._preserve_env = preserve_env + self._mesos_containerizer_path = mesos_containerizer_path # create runner state universal_handler = universal_handler or TaskRunnerUniversalHandler @@ -720,7 +723,8 @@ class TaskRunner(object): logger_mode=logger_mode, rotate_log_size=rotate_log_size, rotate_log_backups=rotate_log_backups, - preserve_env=self._preserve_env) + preserve_env=self._preserve_env, + mesos_containerizer_path=self._mesos_containerizer_path) _DEFAULT_LOGGER = Logger() _DEFAULT_ROTATION = RotatePolicy() http://git-wip-us.apache.org/repos/asf/aurora/blob/a071af34/src/main/python/apache/thermos/runner/thermos_runner.py ---------------------------------------------------------------------- diff --git a/src/main/python/apache/thermos/runner/thermos_runner.py b/src/main/python/apache/thermos/runner/thermos_runner.py index 0d06e8e..441bacd 100644 --- a/src/main/python/apache/thermos/runner/thermos_runner.py +++ b/src/main/python/apache/thermos/runner/thermos_runner.py @@ -83,6 +83,15 @@ app.add_option( app.add_option( + "--mesos_containerizer_path", + dest="mesos_containerizer_path", + metavar="PATH", + default=None, + help="The path to the mesos-containerizer executable that will be used to isolate the task's " + "filesystem (if using a filesystem image).") + + +app.add_option( "--preserve_env", dest="preserve_env", default=False, @@ -211,8 +220,8 @@ def proxy_main(args, opts): process_logger_mode=opts.process_logger_mode, rotate_log_size_mb=opts.rotate_log_size_mb, rotate_log_backups=opts.rotate_log_backups, - preserve_env=opts.preserve_env - ) + preserve_env=opts.preserve_env, + mesos_containerizer_path=opts.mesos_containerizer_path) for sig in (signal.SIGUSR1, signal.SIGUSR2): signal.signal(sig, functools.partial(runner_teardown, task_runner)) http://git-wip-us.apache.org/repos/asf/aurora/blob/a071af34/src/test/java/org/apache/aurora/scheduler/mesos/MesosTaskFactoryImplTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/aurora/scheduler/mesos/MesosTaskFactoryImplTest.java b/src/test/java/org/apache/aurora/scheduler/mesos/MesosTaskFactoryImplTest.java index bb8a849..0be5e49 100644 --- a/src/test/java/org/apache/aurora/scheduler/mesos/MesosTaskFactoryImplTest.java +++ b/src/test/java/org/apache/aurora/scheduler/mesos/MesosTaskFactoryImplTest.java @@ -61,6 +61,7 @@ import org.apache.mesos.Protos.Volume.Mode; import org.junit.Before; import org.junit.Test; +import static org.apache.aurora.gen.apiConstants.TASK_FILESYSTEM_MOUNT_POINT; import static org.apache.aurora.scheduler.base.TaskTestUtil.DEV_TIER; import static org.apache.aurora.scheduler.base.TaskTestUtil.REVOCABLE_TIER; import static org.apache.aurora.scheduler.mesos.MesosTaskFactory.MesosTaskFactoryImpl.DEFAULT_PORT_PROTOCOL; @@ -447,12 +448,15 @@ public class MesosTaskFactoryImplTest extends EasyMockTest { assertEquals( ContainerInfo.newBuilder() .setType(Type.MESOS) - .setMesos(MesosInfo.newBuilder().setImage( - Protos.Image.newBuilder() + .setMesos(MesosInfo.newBuilder()) + .addAllVolumes(EXECUTOR_SETTINGS_WITH_VOLUMES.getExecutorConfig().getVolumeMounts()) + .addVolumes(Volume.newBuilder() + .setContainerPath(TASK_FILESYSTEM_MOUNT_POINT) + .setImage(Protos.Image.newBuilder() .setType(Protos.Image.Type.DOCKER) .setDocker(Protos.Image.Docker.newBuilder() - .setName(imageName + ":" + imageTag)))) - .addAllVolumes(EXECUTOR_SETTINGS_WITH_VOLUMES.getExecutorConfig().getVolumeMounts()) + .setName(imageName + ":" + imageTag))) + .setMode(Mode.RO)) .build(), task.getExecutor().getContainer()); } @@ -480,13 +484,16 @@ public class MesosTaskFactoryImplTest extends EasyMockTest { assertEquals( ContainerInfo.newBuilder() .setType(Type.MESOS) - .setMesos(MesosInfo.newBuilder().setImage( - Protos.Image.newBuilder() + .setMesos(MesosInfo.newBuilder()) + .addAllVolumes(EXECUTOR_SETTINGS_WITH_VOLUMES.getExecutorConfig().getVolumeMounts()) + .addVolumes(Volume.newBuilder() + .setContainerPath(TASK_FILESYSTEM_MOUNT_POINT) + .setImage(Protos.Image.newBuilder() .setType(Protos.Image.Type.APPC) .setAppc(Protos.Image.Appc.newBuilder() .setName(imageName) - .setId(imageId)))) - .addAllVolumes(EXECUTOR_SETTINGS_WITH_VOLUMES.getExecutorConfig().getVolumeMounts()) + .setId(imageId))) + .setMode(Mode.RO)) .build(), task.getExecutor().getContainer()); } http://git-wip-us.apache.org/repos/asf/aurora/blob/a071af34/src/test/python/apache/aurora/executor/common/test_sandbox.py ---------------------------------------------------------------------- diff --git a/src/test/python/apache/aurora/executor/common/test_sandbox.py b/src/test/python/apache/aurora/executor/common/test_sandbox.py index 63f46e2..ce989b1 100644 --- a/src/test/python/apache/aurora/executor/common/test_sandbox.py +++ b/src/test/python/apache/aurora/executor/common/test_sandbox.py @@ -12,7 +12,10 @@ # limitations under the License. # +import grp import os +import pwd +import subprocess import mock import pytest @@ -21,6 +24,7 @@ from twitter.common.contextutil import temporary_dir from apache.aurora.executor.common.sandbox import ( DefaultSandboxProvider, DirectorySandbox, + DockerDirectorySandbox, FileSystemImageSandbox ) @@ -111,16 +115,16 @@ def test_create_ioerror(chown): @mock.patch('os.makedirs') -def test_filesystem_image_sandbox_create_ioerror(makedirs): +def test_docker_directory_sandbox_create_ioerror(makedirs): makedirs.side_effect = IOError('Disk is borked') with mock.patch.dict('os.environ', { - FileSystemImageSandbox.MESOS_DIRECTORY_ENV_VARIABLE: 'some-directory', - FileSystemImageSandbox.MESOS_SANDBOX_ENV_VARIABLE: 'some-sandbox' + DockerDirectorySandbox.MESOS_DIRECTORY_ENV_VARIABLE: 'some-directory', + DockerDirectorySandbox.MESOS_SANDBOX_ENV_VARIABLE: 'some-sandbox' }): with temporary_dir() as d: real_path = os.path.join(d, 'sandbox') - ds = FileSystemImageSandbox(real_path) + ds = DockerDirectorySandbox(real_path) with pytest.raises(DirectorySandbox.CreationError): ds.create() @@ -135,3 +139,52 @@ def test_destroy_ioerror(): shutil_rmtree.side_effect = IOError('What even are you doing?') with pytest.raises(DirectorySandbox.DeletionError): ds.destroy() + + +def assert_create_user_and_group(mock_check_call, gid_exists, uid_exists): + mock_pwent = pwd.struct_passwd(( + 'someuser', # login name + 'hunter2', # password + 834, # uid + 835, # gid + 'Some User', # user name + '/home/someuser', # home directory + '/bin/sh')) # login shell + + mock_grent = grp.struct_group(( + 'users', # group name + '*', # password + 835, # gid + ['someuser'])) # members + + + exception = subprocess.CalledProcessError( + returncode=FileSystemImageSandbox._USER_OR_GROUP_ID_EXISTS, + cmd='some command', + output=None) + + mock_check_call.side_effect = [ + None if gid_exists else exception, + None if uid_exists else exception] + + with temporary_dir() as d: + with mock.patch.object( + FileSystemImageSandbox, + 'get_user_and_group', + return_value=(mock_pwent, mock_grent)): + + sandbox = FileSystemImageSandbox(os.path.join(d, 'sandbox'), user='someuser') + sandbox._create_user_and_group_in_taskfs() + + assert len(mock_check_call.mock_calls) == 2 + +@mock.patch('subprocess.check_call') +@mock.patch.dict(os.environ, {'MESOS_DIRECTORY': '/some/path'}) +def test_uid_exists(mock_check_call): + assert_create_user_and_group(mock_check_call, False, True) + + +@mock.patch('subprocess.check_call') +@mock.patch.dict(os.environ, {'MESOS_DIRECTORY': '/some/path'}) +def test_gid_exists(mock_check_call): + assert_create_user_and_group(mock_check_call, True, False) http://git-wip-us.apache.org/repos/asf/aurora/blob/a071af34/src/test/python/apache/thermos/core/test_process.py ---------------------------------------------------------------------- diff --git a/src/test/python/apache/thermos/core/test_process.py b/src/test/python/apache/thermos/core/test_process.py index 77f644c..759f783 100644 --- a/src/test/python/apache/thermos/core/test_process.py +++ b/src/test/python/apache/thermos/core/test_process.py @@ -23,7 +23,7 @@ import time import mock import pytest from twitter.common.contextutil import mutable_sys, temporary_dir -from twitter.common.dirutil import safe_mkdir +from twitter.common.dirutil import chmod_plus_x, safe_mkdir from twitter.common.quantity import Amount, Data from twitter.common.recordio import ThriftRecordReader @@ -100,6 +100,43 @@ def test_simple_process(): assert_log_content(taskpath, 'stdout', 'hello world\n') +@mock.patch.dict(os.environ, {'MESOS_DIRECTORY': '/some/path'}) +def test_simple_process_filesystem_isolator(): + with temporary_dir() as td: + taskpath = make_taskpath(td) + sandbox = setup_sandbox(td, taskpath) + + test_isolator_path = os.path.join(td, 'fake-mesos-containerier') + with open(test_isolator_path, 'w') as fd: + # We use a fake version of the mesos-containerizer binary that just echoes out its args so + # we can assert on them in the process's output. + fd.write('\n'.join([ + '#!/bin/sh', + 'echo "$@"' + ])) + + fd.close() + + chmod_plus_x(test_isolator_path) + + p = TestProcess( + 'process', + 'echo hello world', + 0, + taskpath, + sandbox, + mesos_containerizer_path=test_isolator_path) + p.start() + + rc = wait_for_rc(taskpath.getpath('process_checkpoint')) + assert rc == 0 + assert_log_content( + taskpath, + 'stdout', + 'launch --unshare_namespace_mnt --rootfs=/some/path/taskfs --user=None ' + '--command={"shell":true,"value":"echo hello world"}\n') + + @mock.patch('os.chown') @mock.patch('os.setgroups') @mock.patch('os.setgid') http://git-wip-us.apache.org/repos/asf/aurora/blob/a071af34/src/test/sh/org/apache/aurora/e2e/Dockerfile ---------------------------------------------------------------------- diff --git a/src/test/sh/org/apache/aurora/e2e/Dockerfile b/src/test/sh/org/apache/aurora/e2e/Dockerfile deleted file mode 100644 index 1b91f02..0000000 --- a/src/test/sh/org/apache/aurora/e2e/Dockerfile +++ /dev/null @@ -1,21 +0,0 @@ -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -FROM python:2.7 - -# The mesos containerizer does not auto-create mount points, so we must initialize them manually. -# TODO(jcohen): Remove this mkdir when https://issues.apache.org/jira/browse/MESOS-5229 is resolved. -RUN mkdir -p /home/vagrant/aurora/examples/vagrant/config - -COPY http_example.py /tmp/ http://git-wip-us.apache.org/repos/asf/aurora/blob/a071af34/src/test/sh/org/apache/aurora/e2e/Dockerfile.netcat ---------------------------------------------------------------------- diff --git a/src/test/sh/org/apache/aurora/e2e/Dockerfile.netcat b/src/test/sh/org/apache/aurora/e2e/Dockerfile.netcat new file mode 100644 index 0000000..c8b2f46 --- /dev/null +++ b/src/test/sh/org/apache/aurora/e2e/Dockerfile.netcat @@ -0,0 +1,18 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +FROM buildpack-deps:jessie + +RUN apt-get update && apt-get install -y netcat-openbsd +COPY run-server.sh /usr/local/bin http://git-wip-us.apache.org/repos/asf/aurora/blob/a071af34/src/test/sh/org/apache/aurora/e2e/Dockerfile.python ---------------------------------------------------------------------- diff --git a/src/test/sh/org/apache/aurora/e2e/Dockerfile.python b/src/test/sh/org/apache/aurora/e2e/Dockerfile.python new file mode 100644 index 0000000..86eac35 --- /dev/null +++ b/src/test/sh/org/apache/aurora/e2e/Dockerfile.python @@ -0,0 +1,17 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +FROM python:2.7 + +COPY http_example.py /tmp/ \ No newline at end of file http://git-wip-us.apache.org/repos/asf/aurora/blob/a071af34/src/test/sh/org/apache/aurora/e2e/http/http_example.aurora ---------------------------------------------------------------------- diff --git a/src/test/sh/org/apache/aurora/e2e/http/http_example.aurora b/src/test/sh/org/apache/aurora/e2e/http/http_example.aurora index bf6ef69..b69ddf1 100644 --- a/src/test/sh/org/apache/aurora/e2e/http/http_example.aurora +++ b/src/test/sh/org/apache/aurora/e2e/http/http_example.aurora @@ -48,6 +48,15 @@ test_task = Task( processes = [echo_ports, stage_server, run_server], constraints = order(echo_ports, stage_server, run_server)) +no_python_task = Task( + name = 'http_example_no_python', + resources = Resources(cpu=0.4, ram=32*MB, disk=64*MB), + processes = [ + echo_ports, + Process(name='run_server', cmdline='run-server.sh {{thermos.ports[http]}}'), + ], + constraints = order(echo_ports, run_server)) + update_config = UpdateConfig(watch_secs=10, batch_size=2) health_check_config = HealthCheckConfig(initial_interval_secs=5, interval_secs=1) @@ -78,10 +87,11 @@ jobs = [ container = Container(docker=Docker(image = 'http_example')) ).bind(profile=ContainerProfile), job( - name = 'http_example_appc', - container = Mesos(image=AppcImage(name='http_example', image_id='{{appc_image_id}}')) - ).bind(profile=ContainerProfile), - job( name = 'http_example_gpu' - ).bind(profile=GpuProfile) + ).bind(profile=GpuProfile), + job( + name = 'http_example_appc', + container = Mesos(image=AppcImage(name='http_example_netcat', image_id='{{appc_image_id}}')), + task = no_python_task + ).bind(profile=DefaultProfile()) ] http://git-wip-us.apache.org/repos/asf/aurora/blob/a071af34/src/test/sh/org/apache/aurora/e2e/http/http_example_bad_healthcheck.aurora ---------------------------------------------------------------------- diff --git a/src/test/sh/org/apache/aurora/e2e/http/http_example_bad_healthcheck.aurora b/src/test/sh/org/apache/aurora/e2e/http/http_example_bad_healthcheck.aurora index edeafbe..f80f2b1 100644 --- a/src/test/sh/org/apache/aurora/e2e/http/http_example_bad_healthcheck.aurora +++ b/src/test/sh/org/apache/aurora/e2e/http/http_example_bad_healthcheck.aurora @@ -45,6 +45,11 @@ test_task = Task( constraints = order(stage_server, run_server) ) +no_python_task = Task( + name = 'http_example_no_python', + resources = Resources(cpu=0.4, ram=32*MB, disk=64*MB), + processes = [Process(name='run_server', cmdline='run-server.sh {{thermos.ports[http]}}')]) + update_config = UpdateConfig(watch_secs=10, batch_size=2) # "I am going to fail" config. shell_config = ShellHealthChecker( @@ -84,10 +89,11 @@ jobs = [ container = Container(docker=Docker(image = 'http_example')) ).bind(profile=ContainerProfile), job( - name = 'http_example_appc', - container = Mesos(image=AppcImage(name='http_example', image_id='{{appc_image_id}}')) - ).bind(profile=ContainerProfile), - job( name = 'http_example_gpu' - ).bind(profile=GpuProfile) + ).bind(profile=GpuProfile), + job( + name = 'http_example_appc', + container = Mesos(image=AppcImage(name='http_example_netcat', image_id='{{appc_image_id}}')), + task = no_python_task + ).bind(profile=DefaultProfile()) ] http://git-wip-us.apache.org/repos/asf/aurora/blob/a071af34/src/test/sh/org/apache/aurora/e2e/http/http_example_updated.aurora ---------------------------------------------------------------------- diff --git a/src/test/sh/org/apache/aurora/e2e/http/http_example_updated.aurora b/src/test/sh/org/apache/aurora/e2e/http/http_example_updated.aurora index 9569eec..4a95bc3 100644 --- a/src/test/sh/org/apache/aurora/e2e/http/http_example_updated.aurora +++ b/src/test/sh/org/apache/aurora/e2e/http/http_example_updated.aurora @@ -42,6 +42,11 @@ test_task = SequentialTask( resources = Resources(cpu=0.4, ram=34*MB, disk=64*MB, gpu='{{profile.gpu}}'), processes = [stage_server, run_server]) +no_python_task = Task( + name = 'http_example_no_python', + resources = Resources(cpu=0.4, ram=32*MB, disk=64*MB), + processes = [Process(name='run_server', cmdline='run-server.sh {{thermos.ports[http]}}')]) + update_config = UpdateConfig(watch_secs=10, batch_size=3) health_check_config = HealthCheckConfig(initial_interval_secs=5, interval_secs=1) @@ -70,10 +75,11 @@ jobs = [ container = Container(docker=Docker(image = 'http_example')) ).bind(profile=ContainerProfile), job( - name = 'http_example_appc', - container = Mesos(image=AppcImage(name='http_example', image_id='{{appc_image_id}}')) - ).bind(profile=ContainerProfile), - job( name = 'http_example_gpu' - ).bind(profile=GpuProfile) + ).bind(profile=GpuProfile), + job( + name = 'http_example_appc', + container = Mesos(image=AppcImage(name='http_example_netcat', image_id='{{appc_image_id}}')), + task = no_python_task + ).bind(profile=DefaultProfile()) ] http://git-wip-us.apache.org/repos/asf/aurora/blob/a071af34/src/test/sh/org/apache/aurora/e2e/run-server.sh ---------------------------------------------------------------------- diff --git a/src/test/sh/org/apache/aurora/e2e/run-server.sh b/src/test/sh/org/apache/aurora/e2e/run-server.sh new file mode 100755 index 0000000..7693988 --- /dev/null +++ b/src/test/sh/org/apache/aurora/e2e/run-server.sh @@ -0,0 +1,7 @@ +#!/bin/sh + +echo "Starting up server..." +while true +do + echo -e "HTTP/1.1 200 OK\n\n Hello from a filesystem image" | nc -l "$1" +done http://git-wip-us.apache.org/repos/asf/aurora/blob/a071af34/src/test/sh/org/apache/aurora/e2e/test_end_to_end.sh ---------------------------------------------------------------------- diff --git a/src/test/sh/org/apache/aurora/e2e/test_end_to_end.sh b/src/test/sh/org/apache/aurora/e2e/test_end_to_end.sh index 47bd94d..0404d0e 100755 --- a/src/test/sh/org/apache/aurora/e2e/test_end_to_end.sh +++ b/src/test/sh/org/apache/aurora/e2e/test_end_to_end.sh @@ -256,11 +256,11 @@ test_run() { echo >> ~/.ssh/authorized_keys cat ${_ssh_key}.pub >> ~/.ssh/authorized_keys - # Using the sandbox contents as a proxy for functioning SSH. List sandbox contents, we expect - # 3 instances of the same thing - our python script. - sandbox_contents=$(aurora task run $_jobkey 'ls' | awk '{print $2}' | sort | uniq -c) + # Using the sandbox contents as a proxy for functioning SSH. List sandbox contents, looking for + # the .logs directory. We expect to find 3 instances. + sandbox_contents=$(aurora task run $_jobkey 'ls -a' | awk '{print $2}' | grep ".logs" | sort | uniq -c) echo "$sandbox_contents" - [[ "$sandbox_contents" = " 3 http_example.py" ]] + [[ "$sandbox_contents" = " 3 .logs" ]] } test_kill() { @@ -404,14 +404,15 @@ test_appc() { pushd "$TEMP_PATH" # build the appc image from the docker image - docker save -o http_example-latest.tar http_example - docker2aci http_example-latest.tar + sudo docker build -t http_example_netcat -f "${TEST_ROOT}/Dockerfile.netcat" ${TEST_ROOT} + docker save -o http_example_netcat-latest.tar http_example_netcat + docker2aci http_example_netcat-latest.tar - APPC_IMAGE_ID="sha512-$(sha512sum http_example-latest.aci | awk '{print $1}')" + APPC_IMAGE_ID="sha512-$(sha512sum http_example_netcat-latest.aci | awk '{print $1}')" APPC_IMAGE_DIRECTORY="/tmp/mesos/images/appc/images/$APPC_IMAGE_ID" sudo mkdir -p "$APPC_IMAGE_DIRECTORY" - sudo tar -xf http_example-latest.aci -C "$APPC_IMAGE_DIRECTORY" + sudo tar -xf http_example_netcat-latest.aci -C "$APPC_IMAGE_DIRECTORY" # This restart is necessary for mesos to pick up the image from the local store. sudo restart mesos-slave @@ -480,10 +481,9 @@ test_http_example_basic "${TEST_JOB_REVOCABLE_ARGS[@]}" test_http_example_basic "${TEST_JOB_GPU_ARGS[@]}" # build the test docker image -sudo docker build -t http_example ${TEST_ROOT} +sudo docker build -t http_example -f "${TEST_ROOT}/Dockerfile.python" ${TEST_ROOT} test_http_example "${TEST_JOB_DOCKER_ARGS[@]}" -# This test relies on the docker image having been built above. test_appc test_admin "${TEST_ADMIN_ARGS[@]}"
