Repository: aurora Updated Branches: refs/heads/master a45952785 -> f3cbc399e
Adding oversubscription e2e tests. Bugs closed: AURORA-1464 Reviewed at https://reviews.apache.org/r/37926/ Project: http://git-wip-us.apache.org/repos/asf/aurora/repo Commit: http://git-wip-us.apache.org/repos/asf/aurora/commit/f3cbc399 Tree: http://git-wip-us.apache.org/repos/asf/aurora/tree/f3cbc399 Diff: http://git-wip-us.apache.org/repos/asf/aurora/diff/f3cbc399 Branch: refs/heads/master Commit: f3cbc399ef85f7976ab984c83a43b113a9db0e83 Parents: a459527 Author: Maxim Khutornenko <[email protected]> Authored: Wed Sep 2 21:23:51 2015 -0700 Committer: Maxim Khutornenko <[email protected]> Committed: Wed Sep 2 21:23:51 2015 -0700 ---------------------------------------------------------------------- examples/vagrant/upstart/aurora-scheduler.conf | 3 +- examples/vagrant/upstart/mesos-slave.conf | 15 ++++- .../apache/aurora/scheduler/http/Offers.java | 3 + src/main/python/apache/aurora/client/config.py | 11 ---- .../python/apache/aurora/client/test_config.py | 15 ----- .../apache/aurora/e2e/http/http_example.aurora | 25 +++++++- .../aurora/e2e/http/http_example_docker.aurora | 53 ---------------- .../e2e/http/http_example_docker_updated.aurora | 53 ---------------- .../aurora/e2e/http/http_example_updated.aurora | 25 +++++++- .../sh/org/apache/aurora/e2e/test_end_to_end.sh | 65 ++++++++++++-------- 10 files changed, 102 insertions(+), 166 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/aurora/blob/f3cbc399/examples/vagrant/upstart/aurora-scheduler.conf ---------------------------------------------------------------------- diff --git a/examples/vagrant/upstart/aurora-scheduler.conf b/examples/vagrant/upstart/aurora-scheduler.conf index e909451..f710295 100644 --- a/examples/vagrant/upstart/aurora-scheduler.conf +++ b/examples/vagrant/upstart/aurora-scheduler.conf @@ -45,4 +45,5 @@ exec bin/aurora-scheduler \ -use_beta_db_task_store=true \ -shiro_ini_path=etc/shiro.example.ini \ -enable_h2_console=true \ - -tier_config=/home/vagrant/aurora/src/test/resources/org/apache/aurora/scheduler/tiers-example.json + -tier_config=/home/vagrant/aurora/src/test/resources/org/apache/aurora/scheduler/tiers-example.json \ + -receive_revocable_resources=true \ No newline at end of file http://git-wip-us.apache.org/repos/asf/aurora/blob/f3cbc399/examples/vagrant/upstart/mesos-slave.conf ---------------------------------------------------------------------- diff --git a/examples/vagrant/upstart/mesos-slave.conf b/examples/vagrant/upstart/mesos-slave.conf index 2b6a606..9af680e 100644 --- a/examples/vagrant/upstart/mesos-slave.conf +++ b/examples/vagrant/upstart/mesos-slave.conf @@ -29,4 +29,17 @@ exec /usr/sbin/mesos-slave --master=zk://$ZK_HOST:2181/mesos/master \ --attributes="host:$MY_HOST;rack:a" \ --resources="cpus:4;mem:1024;disk:20000" \ --work_dir="/var/lib/mesos" \ - --containerizers=docker,mesos + --containerizers=docker,mesos \ + --resource_estimator="org_apache_mesos_FixedResourceEstimator" \ + --modules='{ + "libraries": { + "file": "/usr/lib/libfixed_resource_estimator.so", + "modules": { + "name": "org_apache_mesos_FixedResourceEstimator", + "parameters": { + "key": "resources", + "value": "cpus:3" + } + } + } + }' http://git-wip-us.apache.org/repos/asf/aurora/blob/f3cbc399/src/main/java/org/apache/aurora/scheduler/http/Offers.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/aurora/scheduler/http/Offers.java b/src/main/java/org/apache/aurora/scheduler/http/Offers.java index 4329ce1..15f1582 100644 --- a/src/main/java/org/apache/aurora/scheduler/http/Offers.java +++ b/src/main/java/org/apache/aurora/scheduler/http/Offers.java @@ -113,6 +113,9 @@ public class Offers { if (resource.hasSet()) { builder.put("set", resource.getSet().getItemList()); } + if (resource.hasRevocable()) { + builder.put("revocable", "true"); + } return builder.build(); } }; http://git-wip-us.apache.org/repos/asf/aurora/blob/f3cbc399/src/main/python/apache/aurora/client/config.py ---------------------------------------------------------------------- diff --git a/src/main/python/apache/aurora/client/config.py b/src/main/python/apache/aurora/client/config.py index f5ac084..2fc1255 100644 --- a/src/main/python/apache/aurora/client/config.py +++ b/src/main/python/apache/aurora/client/config.py @@ -22,8 +22,6 @@ import math import re import sys -from pystachio import Empty - from apache.aurora.client import binding_helper from apache.aurora.client.base import die from apache.aurora.config import AuroraConfig @@ -57,7 +55,6 @@ def _validate_announce_configuration(config): STAGING_RE = re.compile(r'^staging\d*$') -#TODO(maxim): Merge env and tier and move definitions to scheduler: AURORA-1443. def __validate_env(name, config_name): if STAGING_RE.match(name): return @@ -71,13 +68,6 @@ def _validate_environment_name(config): __validate_env(env_name, 'Environment') -def _validate_tier(config): - tier_raw = config.raw().tier() - tier_name = str(tier_raw) if tier_raw is not Empty else None - if tier_name is not None: - __validate_env(tier_name, 'Tier') - - UPDATE_CONFIG_MAX_FAILURES_ERROR = ''' max_total_failures in update_config must be lesser than the job size. Based on your job size (%s) you should use max_total_failures <= %s. @@ -128,7 +118,6 @@ def validate_config(config, env=None): _validate_update_config(config) _validate_announce_configuration(config) _validate_environment_name(config) - _validate_tier(config) class GlobalHookRegistry(object): http://git-wip-us.apache.org/repos/asf/aurora/blob/f3cbc399/src/test/python/apache/aurora/client/test_config.py ---------------------------------------------------------------------- diff --git a/src/test/python/apache/aurora/client/test_config.py b/src/test/python/apache/aurora/client/test_config.py index 37459f5..b1a3c18 100644 --- a/src/test/python/apache/aurora/client/test_config.py +++ b/src/test/python/apache/aurora/client/test_config.py @@ -157,21 +157,6 @@ def test_environment_names(): config._validate_environment_name(AuroraConfig(base_job(environment=env_name))) -def test_tier_names(): - base_job = Job( - name='hello_world', role='john_doe', cluster='test-cluster', - task=Task(name='main', processes=[])) - - # Make sure empty value does not raise. - config._validate_tier(AuroraConfig(base_job)) - - for tier in GOOD_ENV: - config._validate_tier(AuroraConfig(base_job(tier=tier))) - for tier in BAD_ENV: - with pytest.raises(ValueError): - config._validate_tier(AuroraConfig(base_job(tier=tier))) - - def test_dedicated_portmap(): base_job = Job( name='hello_world', role='john_doe', cluster='test-cluster', http://git-wip-us.apache.org/repos/asf/aurora/blob/f3cbc399/src/test/sh/org/apache/aurora/e2e/http/http_example.aurora ---------------------------------------------------------------------- diff --git a/src/test/sh/org/apache/aurora/e2e/http/http_example.aurora b/src/test/sh/org/apache/aurora/e2e/http/http_example.aurora index c1a10d8..d7bf108 100644 --- a/src/test/sh/org/apache/aurora/e2e/http/http_example.aurora +++ b/src/test/sh/org/apache/aurora/e2e/http/http_example.aurora @@ -14,13 +14,15 @@ import getpass +DEFAULT_CMD = 'cp /vagrant/src/test/sh/org/apache/aurora/e2e/http_example.py .' + run_server = Process( name = 'run_server', cmdline = 'python http_example.py {{thermos.ports[http]}}') stage_server = Process( name = 'stage_server', - cmdline = "cp /vagrant/src/test/sh/org/apache/aurora/e2e/http_example.py ." + cmdline = '{{cmd}}' ) test_task = Task( @@ -33,7 +35,6 @@ update_config = UpdateConfig(watch_secs=10, batch_size=2) health_check_config = HealthCheckConfig(initial_interval_secs=5, interval_secs=1) job = Service( - name = 'http_example', cluster = 'devcluster', instances = 2, update_config = update_config, @@ -49,4 +50,22 @@ job = Service( announce = Announcer(), ) -jobs = [ job ] +jobs = [ + job( + name = 'http_example' + ).bind( + cmd = DEFAULT_CMD + ), + job( + name = 'http_example_revocable', + tier = 'revocable' + ).bind( + cmd = DEFAULT_CMD + ), + job( + name = 'http_example_docker', + container = Container(docker=Docker(image = 'http_example')) + ).bind( + cmd = 'cp /tmp/http_example.py .' + ) +] http://git-wip-us.apache.org/repos/asf/aurora/blob/f3cbc399/src/test/sh/org/apache/aurora/e2e/http/http_example_docker.aurora ---------------------------------------------------------------------- diff --git a/src/test/sh/org/apache/aurora/e2e/http/http_example_docker.aurora b/src/test/sh/org/apache/aurora/e2e/http/http_example_docker.aurora deleted file mode 100644 index 870b3e6..0000000 --- a/src/test/sh/org/apache/aurora/e2e/http/http_example_docker.aurora +++ /dev/null @@ -1,53 +0,0 @@ -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import getpass - -run_server = Process( - name = 'run_server', - cmdline = 'python http_example.py {{thermos.ports[http]}}') - -stage_server = Process( - name = 'stage_server', - cmdline = "cp /tmp/http_example.py ." -) - -test_task = Task( - name = 'http_example', - resources = Resources(cpu=0.5, ram=32*MB, disk=64*MB), - processes = [stage_server, run_server], - constraints = order(stage_server, run_server)) - -update_config = UpdateConfig(watch_secs=10, batch_size=2) -health_check_config = HealthCheckConfig(initial_interval_secs=5, interval_secs=1) - -job = Service( - name = 'http_example_docker', - cluster = 'devcluster', - instances = 2, - update_config = update_config, - health_check_config = health_check_config, - task = test_task, - role = getpass.getuser(), - environment = 'test', - contact = '{{role}}@localhost', - # Since there is only one slave in devcluster allow all instances to run there. - constraints = { - 'host': 'limit:2', - }, - announce = Announcer(), - container = Container(docker=Docker(image = 'http_example')), -) - -jobs = [ job ] http://git-wip-us.apache.org/repos/asf/aurora/blob/f3cbc399/src/test/sh/org/apache/aurora/e2e/http/http_example_docker_updated.aurora ---------------------------------------------------------------------- diff --git a/src/test/sh/org/apache/aurora/e2e/http/http_example_docker_updated.aurora b/src/test/sh/org/apache/aurora/e2e/http/http_example_docker_updated.aurora deleted file mode 100644 index e55aad3..0000000 --- a/src/test/sh/org/apache/aurora/e2e/http/http_example_docker_updated.aurora +++ /dev/null @@ -1,53 +0,0 @@ -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import getpass - -run_server = Process( - name = 'run_server', - cmdline = 'python http_example.py {{thermos.ports[http]}}') - -stage_server = Process( - name = 'stage_server', - cmdline = "cp /tmp/http_example.py ." -) - -test_task = Task( - name = 'http_example', - resources = Resources(cpu=0.5, ram=34*MB, disk=64*MB), - processes = [stage_server, run_server], - constraints = order(stage_server, run_server)) - -update_config = UpdateConfig(watch_secs=10, batch_size=3) -health_check_config = HealthCheckConfig(initial_interval_secs=5, interval_secs=1) - -job = Service( - name = 'http_example_docker', - cluster = 'devcluster', - instances = 3, - update_config = update_config, - health_check_config = health_check_config, - task = test_task, - role = getpass.getuser(), - environment = 'test', - contact = '{{role}}@localhost', - # Since there is only one slave in devcluster allow all instances to run there. - constraints = { - 'host': 'limit:4', - }, - announce = Announcer(), - container = Container(docker=Docker(image = 'http_example')), -) - -jobs = [ job ] http://git-wip-us.apache.org/repos/asf/aurora/blob/f3cbc399/src/test/sh/org/apache/aurora/e2e/http/http_example_updated.aurora ---------------------------------------------------------------------- diff --git a/src/test/sh/org/apache/aurora/e2e/http/http_example_updated.aurora b/src/test/sh/org/apache/aurora/e2e/http/http_example_updated.aurora index 423dd4d..c973966 100644 --- a/src/test/sh/org/apache/aurora/e2e/http/http_example_updated.aurora +++ b/src/test/sh/org/apache/aurora/e2e/http/http_example_updated.aurora @@ -14,13 +14,15 @@ import getpass +DEFAULT_CMD = 'cp /vagrant/src/test/sh/org/apache/aurora/e2e/http_example.py .' + run_server = Process( name = 'run_server', cmdline = 'python http_example.py {{thermos.ports[http]}}') stage_server = Process( name = 'stage_server', - cmdline = "cp /vagrant/src/test/sh/org/apache/aurora/e2e/http_example.py ." + cmdline = '{{cmd}}' ) test_task = Task( @@ -33,7 +35,6 @@ update_config = UpdateConfig(watch_secs=10, batch_size=3) health_check_config = HealthCheckConfig(initial_interval_secs=5, interval_secs=1) job = Service( - name = 'http_example', cluster = 'devcluster', instances = 3, update_config = update_config, @@ -49,4 +50,22 @@ job = Service( announce = Announcer(), ) -jobs = [ job ] +jobs = [ + job( + name = 'http_example' + ).bind( + cmd = DEFAULT_CMD + ), + job( + name = 'http_example_revocable', + tier = 'revocable' + ).bind( + cmd = DEFAULT_CMD + ), + job( + name = 'http_example_docker', + container = Container(docker=Docker(image = 'http_example')) + ).bind( + cmd = 'cp /tmp/http_example.py .' + ) +] http://git-wip-us.apache.org/repos/asf/aurora/blob/f3cbc399/src/test/sh/org/apache/aurora/e2e/test_end_to_end.sh ---------------------------------------------------------------------- diff --git a/src/test/sh/org/apache/aurora/e2e/test_end_to_end.sh b/src/test/sh/org/apache/aurora/e2e/test_end_to_end.sh index 018efbe..1eec20c 100755 --- a/src/test/sh/org/apache/aurora/e2e/test_end_to_end.sh +++ b/src/test/sh/org/apache/aurora/e2e/test_end_to_end.sh @@ -33,7 +33,7 @@ _curl() { curl --silent --fail --retry 4 --retry-delay 10 "$@" ; } tear_down() { set +x # Disable command echo, as this makes it more difficult see which command failed. - for job in http_example http_example_docker; do + for job in http_example http_example_revocable http_example_docker; do aurora job cancel-update devcluster/vagrant/test/$job >/dev/null 2>&1 aurora update abort devcluster/vagrant/test/$job || true >/dev/null 2>&1 aurora job killall --no-batching devcluster/vagrant/test/$job >/dev/null 2>&1 @@ -65,7 +65,7 @@ test_config() { local _config=$1 _jobkey=$2 joblist=$(aurora config list $_config | tr -dc '[[:print:]]') - [[ "$joblist" = "jobs=[$_jobkey]" ]] + [[ "$joblist" = *"$_jobkey"* ]] } test_inspect() { @@ -225,8 +225,9 @@ test_quota() { } test_http_example() { - local _cluster=$1 _role=$2 _env=$3 _job=$4 - local _base_config=$5 _updated_config=$6 + local _cluster=$1 _role=$2 _env=$3 + local _base_config=$4 _updated_config=$5 + local _job=$6 local _jobkey="$_cluster/$_role/$_env/$_job" test_config $_base_config $_jobkey @@ -244,6 +245,17 @@ test_http_example() { test_quota $_cluster $_role } +test_http_revocable_example() { + local _cluster=$1 _role=$2 _env=$3 + local _base_config=$4 + local _job=$6 + local _jobkey="$_cluster/$_role/$_env/$_job" + + test_create $_jobkey $_base_config + test_observer_ui $_cluster $_role $_job + test_kill $_jobkey +} + test_admin() { local _cluster=$1 echo '== Testing admin commands' @@ -256,8 +268,9 @@ restore_netrc() { } test_basic_auth_unauthenticated() { - local _cluster=$1 _role=$2 _env=$3 _job=$4 - local _config=$5 + local _cluster=$1 _role=$2 _env=$3 + local _config=$4 + local _job=$6 local _jobkey="$_cluster/$_role/$_env/$_job" mv ~/.netrc ~/.netrc.bak @@ -280,41 +293,41 @@ TEST_CLUSTER=devcluster TEST_ROLE=vagrant TEST_ENV=test TEST_JOB=http_example -TEST_DOCKER_JOB=http_example_docker -TEST_ARGS=( - $TEST_CLUSTER - $TEST_ROLE - $TEST_ENV - $TEST_JOB - $EXAMPLE_DIR/http_example.aurora - $EXAMPLE_DIR/http_example_updated.aurora - ) - -TEST_ADMIN_ARGS=( - $TEST_CLUSTER -) +TEST_JOB_REVOCABLE=http_example_revocable +TEST_JOB_DOCKER=http_example_docker +TEST_CONFIG_FILE=$EXAMPLE_DIR/http_example.aurora +TEST_CONFIG_UPDATED_FILE=$EXAMPLE_DIR/http_example_updated.aurora -TEST_DOCKER_ARGS=( +BASE_ARGS=( $TEST_CLUSTER $TEST_ROLE $TEST_ENV - $TEST_DOCKER_JOB - $EXAMPLE_DIR/http_example_docker.aurora - $EXAMPLE_DIR/http_example_docker_updated.aurora + $TEST_CONFIG_FILE + $TEST_CONFIG_UPDATED_FILE ) +TEST_JOB_ARGS=("${BASE_ARGS[@]}" "$TEST_JOB") + +TEST_JOB_REVOCABLE_ARGS=("${BASE_ARGS[@]}" "$TEST_JOB_REVOCABLE") + +TEST_JOB_DOCKER_ARGS=("${BASE_ARGS[@]}" "$TEST_JOB_DOCKER") + +TEST_ADMIN_ARGS=($TEST_CLUSTER) + trap collect_result EXIT aurorabuild all test_version -test_http_example "${TEST_ARGS[@]}" +test_http_example "${TEST_JOB_ARGS[@]}" + +test_http_revocable_example "${TEST_JOB_REVOCABLE_ARGS[@]}" # build the test docker image sudo docker build -t http_example ${TEST_ROOT} -test_http_example "${TEST_DOCKER_ARGS[@]}" +test_http_example "${TEST_JOB_DOCKER_ARGS[@]}" test_admin "${TEST_ADMIN_ARGS[@]}" -test_basic_auth_unauthenticated "${TEST_ARGS[@]}" +test_basic_auth_unauthenticated "${TEST_JOB_ARGS[@]}" /vagrant/src/test/sh/org/apache/aurora/e2e/test_kerberos_end_to_end.sh RETCODE=0
