Repository: aurora Updated Branches: refs/heads/master d4ebb56ba -> eb3660d9e
Ensure destination exists when mounting files into a filesystem image. When testing filesystem isolation internally, we ran into an issue where mounting a regular file into the task filesystem failed with exit code 32 since the mount destination did not exist. To account for this, we'll touch an empty file in the taskfs. Reviewed at https://reviews.apache.org/r/55347/ Project: http://git-wip-us.apache.org/repos/asf/aurora/repo Commit: http://git-wip-us.apache.org/repos/asf/aurora/commit/eb3660d9 Tree: http://git-wip-us.apache.org/repos/asf/aurora/tree/eb3660d9 Diff: http://git-wip-us.apache.org/repos/asf/aurora/diff/eb3660d9 Branch: refs/heads/master Commit: eb3660d9e0d7299b189d964e44ceca016954499b Parents: d4ebb56 Author: Joshua Cohen <jco...@apache.org> Authored: Tue Jan 10 16:11:54 2017 -0600 Committer: Joshua Cohen <jco...@apache.org> Committed: Tue Jan 10 16:11:54 2017 -0600 ---------------------------------------------------------------------- examples/vagrant/upstart/aurora-scheduler.conf | 2 +- .../apache/aurora/executor/common/sandbox.py | 11 ++- .../aurora/executor/common/test_sandbox.py | 75 +++++++++++++++++++- .../apache/aurora/e2e/http/http_example.aurora | 6 ++ .../sh/org/apache/aurora/e2e/test_end_to_end.sh | 20 +++++- 5 files changed, 107 insertions(+), 7 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/aurora/blob/eb3660d9/examples/vagrant/upstart/aurora-scheduler.conf ---------------------------------------------------------------------- diff --git a/examples/vagrant/upstart/aurora-scheduler.conf b/examples/vagrant/upstart/aurora-scheduler.conf index e68a801..91f0f0c 100644 --- a/examples/vagrant/upstart/aurora-scheduler.conf +++ b/examples/vagrant/upstart/aurora-scheduler.conf @@ -40,7 +40,7 @@ exec bin/aurora-scheduler \ -native_log_file_path=/var/db/aurora \ -backup_dir=/var/lib/aurora/backups \ -thermos_executor_path=$DIST_DIR/thermos_executor.pex \ - -global_container_mounts=/home/vagrant/aurora/examples/vagrant/config:/home/vagrant/aurora/examples/vagrant/config:ro \ + -global_container_mounts=/home/vagrant/aurora/examples/vagrant/config:/home/vagrant/aurora/examples/vagrant/config:ro,/home/vagrant/aurora/.auroraversion:/home/vagrant/aurora/.auroraversion:ro \ -thermos_executor_flags="--announcer-ensemble localhost:2181 --announcer-zookeeper-auth-config /home/vagrant/aurora/examples/vagrant/config/announcer-auth.json --mesos-containerizer-path=/usr/libexec/mesos/mesos-containerizer" \ -allowed_container_types=MESOS,DOCKER \ -http_authentication_mechanism=BASIC \ http://git-wip-us.apache.org/repos/asf/aurora/blob/eb3660d9/src/main/python/apache/aurora/executor/common/sandbox.py ---------------------------------------------------------------------- diff --git a/src/main/python/apache/aurora/executor/common/sandbox.py b/src/main/python/apache/aurora/executor/common/sandbox.py index 9d6e73c..fe1562e 100644 --- a/src/main/python/apache/aurora/executor/common/sandbox.py +++ b/src/main/python/apache/aurora/executor/common/sandbox.py @@ -21,7 +21,7 @@ import subprocess from abc import abstractmethod, abstractproperty from twitter.common import log -from twitter.common.dirutil import safe_mkdir, safe_rmtree +from twitter.common.dirutil import safe_mkdir, safe_rmtree, touch from twitter.common.lang import Interface from gen.apache.aurora.api.constants import TASK_FILESYSTEM_MOUNT_POINT @@ -279,9 +279,16 @@ class FileSystemImageSandbox(DirectorySandbox): def _mount_paths(self): def do_mount(source, destination): - safe_mkdir(destination) log.info('Mounting %s into task filesystem at %s.' % (source, destination)) + # If we're mounting a file into the task filesystem, the mount call will fail if the mount + # point doesn't exist. In that case we'll create an empty file to mount over. + if os.path.isfile(source) and not os.path.exists(destination): + safe_mkdir(os.path.dirname(destination)) + touch(destination) + else: + safe_mkdir(destination) + # This mount call is meant to mimic what mesos does when mounting into the container. C.f. # https://github.com/apache/mesos/blob/c3228f3c3d1a1b2c145d1377185cfe22da6079eb/src/slave/containerizer/mesos/isolators/filesystem/linux.cpp#L521-L528 subprocess.check_call([ http://git-wip-us.apache.org/repos/asf/aurora/blob/eb3660d9/src/test/python/apache/aurora/executor/common/test_sandbox.py ---------------------------------------------------------------------- diff --git a/src/test/python/apache/aurora/executor/common/test_sandbox.py b/src/test/python/apache/aurora/executor/common/test_sandbox.py index a441d2a..0b4958a 100644 --- a/src/test/python/apache/aurora/executor/common/test_sandbox.py +++ b/src/test/python/apache/aurora/executor/common/test_sandbox.py @@ -323,10 +323,11 @@ def test_group_name_exists(mock_check_call, mock_verify): assert_create_user_and_group(mock_check_call, mock_verify, True, False, True, False) +@mock.patch('os.path.isfile') @mock.patch('subprocess.check_call') @mock.patch('apache.aurora.executor.common.sandbox.safe_mkdir') @mock.patch.dict(os.environ, {'MESOS_DIRECTORY': MOCK_MESOS_DIRECTORY}) -def test_filesystem_sandbox_mounts_paths(mock_safe_mkdir, mock_check_call): +def test_filesystem_sandbox_mounts_paths(mock_safe_mkdir, mock_check_call, mock_isfile): sandbox_mount_point = '/some/mount/point' sandbox_directory = os.path.join(MOCK_MESOS_DIRECTORY, 'sandbox') @@ -337,6 +338,8 @@ def test_filesystem_sandbox_mounts_paths(mock_safe_mkdir, mock_check_call): mounted_volume_paths=['/some/container/path', '/some/other/container/path'], sandbox_mount_point=sandbox_mount_point) + mock_isfile.return_value = False + sandbox._mount_paths() task_fs_path = os.path.join(MOCK_MESOS_DIRECTORY, 'taskfs') @@ -366,10 +369,11 @@ def test_filesystem_sandbox_mounts_paths(mock_safe_mkdir, mock_check_call): ] +@mock.patch('os.path.isfile') @mock.patch('subprocess.check_call') @mock.patch('apache.aurora.executor.common.sandbox.safe_mkdir') @mock.patch.dict(os.environ, {'MESOS_DIRECTORY': MOCK_MESOS_DIRECTORY}) -def test_filesystem_sandbox_no_volumes(mock_safe_mkdir, mock_check_call): +def test_filesystem_sandbox_no_volumes(mock_safe_mkdir, mock_check_call, mock_isfile): sandbox_mount_point = '/some/mount/point' sandbox_directory = os.path.join(MOCK_MESOS_DIRECTORY, 'sandbox'), @@ -380,15 +384,82 @@ def test_filesystem_sandbox_no_volumes(mock_safe_mkdir, mock_check_call): mounted_volume_paths=None, sandbox_mount_point=sandbox_mount_point) + mock_isfile.return_value = False + + sandbox._mount_paths() + + task_fs_path = os.path.join(MOCK_MESOS_DIRECTORY, 'taskfs') + + assert mock_check_call.mock_calls == [ + mock.call([ + 'mount', + '-n', + '--rbind', + sandbox_directory, + os.path.join(task_fs_path, sandbox_mount_point[1:]) + ]) + ] + + +@mock.patch('apache.aurora.executor.common.sandbox.touch') +@mock.patch('os.path.exists') +@mock.patch('os.path.isfile') +@mock.patch('subprocess.check_call') +@mock.patch('apache.aurora.executor.common.sandbox.safe_mkdir') +@mock.patch.dict(os.environ, {'MESOS_DIRECTORY': MOCK_MESOS_DIRECTORY}) +def test_filesystem_sandbox_mounts_paths_source_is_file( + mock_safe_mkdir, + mock_check_call, + mock_isfile, + mock_exists, + mock_touch): + + sandbox_mount_point = '/some/mount/point' + sandbox_directory = os.path.join(MOCK_MESOS_DIRECTORY, 'sandbox') + + sandbox = FileSystemImageSandbox( + sandbox_directory, + user='someuser', + no_create_user=True, + mounted_volume_paths=['/some/path/to/file', '/some/path/to/directory'], + sandbox_mount_point=sandbox_mount_point) + + def is_file_side_effect(arg): + return arg.endswith('file') + + mock_isfile.side_effect = is_file_side_effect + mock_exists.return_value = False + sandbox._mount_paths() task_fs_path = os.path.join(MOCK_MESOS_DIRECTORY, 'taskfs') + destination_path = os.path.join(task_fs_path, 'some/path/to/file') + # we should `touch` the mount point, but only for the file mount, not the directory mount. + assert mock_touch.mock_calls == [ + mock.call(destination_path) + ] + + # we should have mounted the file path we passed in as well as the sandbox directory itself. assert mock_check_call.mock_calls == [ mock.call([ 'mount', '-n', '--rbind', + '/some/path/to/file', + destination_path + ]), + mock.call([ + 'mount', + '-n', + '--rbind', + '/some/path/to/directory', + os.path.join(task_fs_path, 'some/path/to/directory') + ]), + mock.call([ + 'mount', + '-n', + '--rbind', sandbox_directory, os.path.join(task_fs_path, sandbox_mount_point[1:]) ]) http://git-wip-us.apache.org/repos/asf/aurora/blob/eb3660d9/src/test/sh/org/apache/aurora/e2e/http/http_example.aurora ---------------------------------------------------------------------- diff --git a/src/test/sh/org/apache/aurora/e2e/http/http_example.aurora b/src/test/sh/org/apache/aurora/e2e/http/http_example.aurora index 438eb0f..1985f89 100644 --- a/src/test/sh/org/apache/aurora/e2e/http/http_example.aurora +++ b/src/test/sh/org/apache/aurora/e2e/http/http_example.aurora @@ -54,6 +54,11 @@ read_env = Process( cmdline = 'echo "$IT_WORKED"' ) +verify_file_mount = Process( + name = 'verify_file_mount', + cmdline = 'cat /home/vagrant/aurora/.auroraversion' +) + test_task = SequentialTask( name = 'http_example', resources = Resources(cpu=0.5, ram=32*MB, disk=64*MB, gpu='{{profile.gpu}}'), @@ -67,6 +72,7 @@ no_python_task = SequentialTask( setup_env, read_env, echo_ports, + verify_file_mount, Process(name='run_server', cmdline='run-server.sh {{thermos.ports[http]}}'), ] ) http://git-wip-us.apache.org/repos/asf/aurora/blob/eb3660d9/src/test/sh/org/apache/aurora/e2e/test_end_to_end.sh ---------------------------------------------------------------------- diff --git a/src/test/sh/org/apache/aurora/e2e/test_end_to_end.sh b/src/test/sh/org/apache/aurora/e2e/test_end_to_end.sh index 4630937..736d1fc 100755 --- a/src/test/sh/org/apache/aurora/e2e/test_end_to_end.sh +++ b/src/test/sh/org/apache/aurora/e2e/test_end_to_end.sh @@ -34,11 +34,11 @@ tear_down() { set +x # Disable command echo, as this makes it more difficult see which command failed. for job in http_example http_example_watch_secs http_example_revocable http_example_docker http_example_unified_appc http_example_unified_docker; do - aurora update abort devcluster/vagrant/test/$job || true >/dev/null 2>&1 + aurora update abort devcluster/vagrant/test/$job >/dev/null 2>&1 || true aurora job killall --no-batching devcluster/vagrant/test/$job >/dev/null 2>&1 done - sudo mv /etc/aurora/clusters.json.old /etc/aurora/clusters.json || true > /dev/null 2>&1 + sudo mv /etc/aurora/clusters.json.old /etc/aurora/clusters.json >/dev/null 2>&1 || true } collect_result() { @@ -346,6 +346,7 @@ test_http_example() { test_observer_ui $_cluster $_role $_job test_discovery_info $_task_id_prefix $_discovery_name test_thermos_profile $_jobkey + test_file_mount $_cluster $_role $_env $_job test_restart $_jobkey test_update $_jobkey $_updated_config $_cluster $_bind_parameters test_update_fail $_jobkey $_base_config $_cluster $_bad_healthcheck_config $_bind_parameters @@ -476,6 +477,21 @@ test_appc_unified() { [[ "$num_mounts_before" = "$num_mounts_after" ]] } +test_file_mount() { + local _cluster=$1 _role=$2 _env=$3 _job=$4 + + if [[ "$_job" = "$TEST_JOB_UNIFIED_DOCKER" ]]; then + local _jobkey="$_cluster/$_role/$_env/$_job" + + verify_file_mount_output=$(aurora task ssh $_jobkey/0 --command='tail -1 .logs/verify_file_mount/0/stdout' |tr -d '\r\n' 2>/dev/null) + echo "$verify_file_mount_output" + [[ "$verify_file_mount_output" = "$(cat /vagrant/.auroraversion |tr -d '\r\n')" ]] + return $? + fi + + return 0 +} + test_docker_unified() { num_mounts_before=$(mount |wc -l |tr -d '\n')
