Repository: aurora Updated Branches: refs/heads/master e2ea19147 -> c52137e20
Upgrading Vagrant setup from Ubuntu Trusty to Ubuntu Xenial. Deleted Thrift install from build script as it's no longer needed. Changing docker images in e2e tests from Debian Jessie to Debian Stretch for ABI compatibility. Bugs closed: AURORA-1964 Reviewed at https://reviews.apache.org/r/65565/ Project: http://git-wip-us.apache.org/repos/asf/aurora/repo Commit: http://git-wip-us.apache.org/repos/asf/aurora/commit/c52137e2 Tree: http://git-wip-us.apache.org/repos/asf/aurora/tree/c52137e2 Diff: http://git-wip-us.apache.org/repos/asf/aurora/diff/c52137e2 Branch: refs/heads/master Commit: c52137e20bd2863234dc09116e1339364ffed77a Parents: e2ea191 Author: Renan DelValle <re...@apache.org> Authored: Wed Feb 21 09:43:45 2018 -0800 Committer: Renan DelValle <re...@apache.org> Committed: Wed Feb 21 09:43:45 2018 -0800 ---------------------------------------------------------------------- Vagrantfile | 3 +- build-support/packer/README.md | 40 ++++++++++-- build-support/packer/aurora-release.json | 38 +++++++++++ build-support/packer/aurora.json | 2 +- build-support/packer/build.sh | 37 ++++------- docs/getting-started/vagrant.md | 4 +- examples/jobs/hello_docker_image.aurora | 2 +- examples/vagrant/aurorabuild.sh | 13 +--- examples/vagrant/provision-dev-cluster.sh | 16 +++-- .../vagrant/systemd/aurora-executor.service | 31 +++++++++ .../systemd/aurora-scheduler-kerberos.service | 64 +++++++++++++++++++ .../vagrant/systemd/aurora-scheduler.service | 67 ++++++++++++++++++++ .../upstart/aurora-scheduler-kerberos.conf | 56 ---------------- examples/vagrant/upstart/aurora-scheduler.conf | 58 ----------------- .../upstart/aurora-thermos-observer.conf | 22 ------- .../sh/org/apache/aurora/e2e/Dockerfile.netcat | 2 +- .../sh/org/apache/aurora/e2e/Dockerfile.python | 2 +- .../apache/aurora/e2e/http/http_example.aurora | 2 +- .../sh/org/apache/aurora/e2e/test_end_to_end.sh | 16 ++--- 19 files changed, 276 insertions(+), 199 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/aurora/blob/c52137e2/Vagrantfile ---------------------------------------------------------------------- diff --git a/Vagrantfile b/Vagrantfile index 2c4d23a..76dfcac 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -25,7 +25,7 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| config.vm.hostname = "aurora.local" # See build-support/packer/README.md for instructions on updating this box. config.vm.box = "apache-aurora/dev-environment" - config.vm.box_version = "0.0.14" + config.vm.box_version = "0.0.15" config.vm.define "devcluster" do |dev| dev.vm.network :private_network, ip: "192.168.33.7", :auto_config => false @@ -33,6 +33,7 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| dev.vm.provider :virtualbox do |vb| vb.customize ["modifyvm", :id, "--memory", "3072"] vb.customize ["modifyvm", :id, "--natdnshostresolver1", "on"] + vb.customize ["modifyvm", :id, "--cableconnected1", "on"] end dev.vm.provision "shell", path: "examples/vagrant/provision-dev-cluster.sh" end http://git-wip-us.apache.org/repos/asf/aurora/blob/c52137e2/build-support/packer/README.md ---------------------------------------------------------------------- diff --git a/build-support/packer/README.md b/build-support/packer/README.md index 78a66ee..f2b793f 100644 --- a/build-support/packer/README.md +++ b/build-support/packer/README.md @@ -19,10 +19,10 @@ will be helpful to update this box and keep this cost low. 3. Fetch the latest version of our base box - $ vagrant box update --box ubuntu/trusty64 + $ vagrant box update --box bento/ubuntu-16.04 The box will be stored in version-specific directories under - `~/.vagrant.d/boxes/ubuntu-VAGRANTSLASH-trusty64/`. Find the path to the `.ovf` file for the + `~/.vagrant.d/boxes/bento-VAGRANTSLASH-ubuntu-16.04/`. Find the path to the `.ovf` file for the latest version of the box. In the following step, this path will be referred to as `$UBUNTU_OVF`. @@ -58,6 +58,36 @@ will be helpful to update this box and keep this cost low. At this point, you can use the box as normal to run integration tests. 6. Upload the box to Vagrant Cloud - Our boxes are stored [here](https://atlas.hashicorp.com/apache-aurora/boxes/dev-environment). - You must have committer access to upload a dev image box, please - ask in d...@aurora.apache.org if you would like to contribute. + Our boxes are stored in [Vagrant Cloud](https://vagrantcloud.com/apache-aurora/dev-environment). + + In order to upload a new version of our box, you must have committer access to upload + a dev image box, please ask in d...@aurora.apache.org, our IRC channel, or our Slack Channel + if you would like to contribute. More info can be found by visiting our + [community page](http://aurora.apache.org/community/). + + Once you have access to our Vagrant Cloud organization, a token can be generated by + going to your [security settings](https://app.vagrantup.com/settings/security). + Store the token in a safe place as it will be needed for future submissions. + + Next, three environmental variables must be set: `$UBUNTU_OVF`, `$VAGRANT_CLOUD_TOKEN`, + and `$BOX_VERSION`. + + $ export UBUNTU_OVF=<Location of base image (.ovf) on local machine> + $ export VAGRANT_CLOUD_TOKEN=<API Token from Hashicorp> + $ export BOX_VERSION=<SemVer to be given to this box> + + **Make sure the variables are set correctly before proceeding as a mistake can cause + the very time consuming process to fail or clobber a previous box.** + + $ env | grep -E "UBUNTU_OVF|VAGRANT_CLOUD_TOKEN|BOX_VERSION" + + Then finally run the release packer configuration which will upload the vagrant box. + + $ packer build aurora-release.json + + Note: This process will rebuild the box and upload it to the Vagrant Cloud. Unfortunately, + there currently is no way to skip the build step as the output from the first post-processor + (Vagrant) is required for the second (Vagrant-Cloud). + + You may now change the version in [`Vagrantfile`](../../Vagrantfile) to the one specified in + `$BOX_VERSION` and commit the change. \ No newline at end of file http://git-wip-us.apache.org/repos/asf/aurora/blob/c52137e2/build-support/packer/aurora-release.json ---------------------------------------------------------------------- diff --git a/build-support/packer/aurora-release.json b/build-support/packer/aurora-release.json new file mode 100644 index 0000000..fa8816e --- /dev/null +++ b/build-support/packer/aurora-release.json @@ -0,0 +1,38 @@ +{ + "variables": { + "base_box_ovf": "{{env `UBUNTU_OVF`}}", + "cloud_token": "{{env `VAGRANT_CLOUD_TOKEN`}}", + "version": "{{env `BOX_VERSION`}}" + }, + "builders": [{ + "type": "virtualbox-ovf", + "source_path": "{{user `base_box_ovf`}}", + "ssh_username": "vagrant", + "ssh_password": "vagrant", + "shutdown_command": "echo 'vagrant' | sudo -S shutdown -P now", + "headless": true, + "vboxmanage": [ + ["modifyvm", "{{.Name}}", "--memory", "4096"], + ["modifyvm", "{{.Name}}", "--cpus", "2"] + ] + }], + "provisioners": [{ + "type": "shell", + "execute_command": "echo 'vagrant' | {{.Vars}} sudo -S -E bash '{{.Path}}'", + "scripts": [ + "build.sh" + ] + }], + "post-processors": [[ + { + "type": "vagrant" + }, + { + "type": "vagrant-cloud", + "box_tag": "apache-aurora/dev-environment", + "access_token": "{{user `cloud_token`}}", + "version": "{{user `version`}}" + } + ] + ] +} http://git-wip-us.apache.org/repos/asf/aurora/blob/c52137e2/build-support/packer/aurora.json ---------------------------------------------------------------------- diff --git a/build-support/packer/aurora.json b/build-support/packer/aurora.json index 043b3d1..da27917 100644 --- a/build-support/packer/aurora.json +++ b/build-support/packer/aurora.json @@ -1,6 +1,6 @@ { "variables": { - "base_box_ovf": null + "base_box_ovf": "{{env `UBUNTU_OVF`}}", }, "builders": [{ "type": "virtualbox-ovf", http://git-wip-us.apache.org/repos/asf/aurora/blob/c52137e2/build-support/packer/build.sh ---------------------------------------------------------------------- diff --git a/build-support/packer/build.sh b/build-support/packer/build.sh index 900a7c6..7c36f74 100644 --- a/build-support/packer/build.sh +++ b/build-support/packer/build.sh @@ -20,14 +20,13 @@ set -o verbose readonly MESOS_VERSION=1.4.0 function remove_unused { - # The default ubuntu/trusty64 image includes juju-core, which adds ~300 MB to our image. + # The default bento/ubuntu-16.04 image includes juju-core, which adds ~300 MB to our image. apt-get purge -y --auto-remove juju-core rm -f /home/vagrant/VBoxGuestAdditions.iso } function install_base_packages { - add-apt-repository ppa:openjdk-r/ppa -y apt-get update apt-get -y install \ bison \ @@ -37,6 +36,7 @@ function install_base_packages { libapr1-dev \ libcurl4-nss-dev \ libffi-dev \ + libkrb5-dev \ libsasl2-dev \ libsvn-dev \ openjdk-8-jdk-headless \ @@ -47,23 +47,23 @@ function install_base_packages { } function install_docker { - # Instructions from https://docs.docker.com/engine/installation/linux/ubuntulinux/ - apt-get install -y apt-transport-https ca-certificates - apt-key adv --keyserver hkp://p80.pool.sks-keyservers.net:80 \ - --recv-keys 58118E89F3A912897C070ADBF76221572C52609D - echo 'deb https://apt.dockerproject.org/repo ubuntu-trusty main' \ - > /etc/apt/sources.list.d/docker.list + # Instructions from https://docs.docker.com/engine/installation/linux/docker-ce/ubuntu/ + apt-get install -y apt-transport-https ca-certificates curl software-properties-common + curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - + add-apt-repository \ + "deb [arch=amd64] https://download.docker.com/linux/ubuntu \ + $(lsb_release -cs) \ + stable" apt-get update apt-get -y install \ - linux-image-extra-$(uname -r) \ apparmor \ - docker-engine + docker-ce docker run -d -p 5000:5000 --restart=always --name registry registry:2 } function install_docker2aci { - DOCKER2ACI_VERSION="0.9.3" - GOLANG_VERSION="1.6.2" + DOCKER2ACI_VERSION="0.17.1" + GOLANG_VERSION="1.9.2" TEMP_PATH=$(mktemp -d) pushd "$TEMP_PATH" @@ -111,16 +111,6 @@ function install_mesos { apt-get -y install mesos=${MESOS_VERSION}* } -function install_thrift { - # Install thrift, needed for code generation in the scheduler build. - curl -sSL http://apache.org/dist/thrift/KEYS | gpg --import - - gpg --export --armor 66B778F9 | apt-key add - - echo 'deb http://www.apache.org/dist/thrift/debian 0.10.0 main' \ - > /etc/apt/sources.list.d/thrift.list - apt-get update - apt-get install thrift-compiler=0.10.0 -} - function warm_artifact_cache { # Gradle caches in the user's home directory. Since development commands # are executed by the vagrant user, switch to that user. @@ -140,7 +130,7 @@ function warm_artifact_cache { SVN_ROOT='https://svn.apache.org/repos/asf/aurora/3rdparty' pushd "$THIRD_PARTY_DIR" wget -c \ - ${SVN_ROOT}/ubuntu/trusty64/python/mesos.executor-${MESOS_VERSION}-py2.7-linux-x86_64.egg + ${SVN_ROOT}/ubuntu/xenial64/python/mesos.executor-${MESOS_VERSION}-py2.7-linux-x86_64.egg popd chown -R vagrant:vagrant aurora @@ -161,6 +151,5 @@ install_base_packages install_docker install_docker2aci install_mesos -install_thrift warm_artifact_cache compact_box http://git-wip-us.apache.org/repos/asf/aurora/blob/c52137e2/docs/getting-started/vagrant.md ---------------------------------------------------------------------- diff --git a/docs/getting-started/vagrant.md b/docs/getting-started/vagrant.md index ca4b704..73d0aff 100644 --- a/docs/getting-started/vagrant.md +++ b/docs/getting-started/vagrant.md @@ -148,7 +148,7 @@ Most of the Vagrant related problems can be fixed by the following steps: If that still doesn't solve your problem, make sure to inspect the log files: -* Scheduler: `/var/log/upstart/aurora-scheduler.log` -* Observer: `/var/log/upstart/aurora-thermos-observer.log` +* Scheduler: `sudo journalctl -u aurora-scheduler` +* Observer: `sudo journalctl -u aurora-executor` * Mesos Master: `/var/log/mesos/mesos-master.INFO` (also see `.WARNING` and `.ERROR`) * Mesos Agent: `/var/log/mesos/mesos-slave.INFO` (also see `.WARNING` and `.ERROR`) http://git-wip-us.apache.org/repos/asf/aurora/blob/c52137e2/examples/jobs/hello_docker_image.aurora ---------------------------------------------------------------------- diff --git a/examples/jobs/hello_docker_image.aurora b/examples/jobs/hello_docker_image.aurora index 9cd1e4a..049a147 100644 --- a/examples/jobs/hello_docker_image.aurora +++ b/examples/jobs/hello_docker_image.aurora @@ -34,7 +34,7 @@ jobs = [ role = 'www-data', name = 'hello_docker_image', task = task, - container = Mesos(image=DockerImage(name='debian', tag='jessie')) + container = Mesos(image=DockerImage(name='debian', tag='stretch')) ) ] http://git-wip-us.apache.org/repos/asf/aurora/blob/c52137e2/examples/vagrant/aurorabuild.sh ---------------------------------------------------------------------- diff --git a/examples/vagrant/aurorabuild.sh b/examples/vagrant/aurorabuild.sh index 2e6f075..5eb1822 100755 --- a/examples/vagrant/aurorabuild.sh +++ b/examples/vagrant/aurorabuild.sh @@ -23,15 +23,6 @@ set -o nounset REPO_DIR=/home/vagrant/aurora DIST_DIR=$REPO_DIR/dist -function upstart_update { - # Stop and start is necessary to update a the configuration of - # an upstart job. We'll rarely change the configuration, but - # it's probably better to do this upfront and avoid surprises/confusion. - # Executing true on failure to please bash -e - sudo stop $1 || true - sudo start $1 || true -} - function build_client { ./pants binary src/main/python/apache/aurora/kerberos:kaurora sudo ln -sf $DIST_DIR/kaurora.pex /usr/local/bin/aurora @@ -63,7 +54,7 @@ function build_scheduler { else echo "Replicated log initialization failed with code $? (likely already initialized)." fi - upstart_update aurora-scheduler + sudo systemctl restart aurora-scheduler } function build_executor { @@ -80,7 +71,7 @@ function build_observer { ./pants binary src/main/python/apache/aurora/tools:thermos_observer ./pants binary src/main/python/apache/aurora/tools:thermos sudo ln -sf $DIST_DIR/thermos.pex /usr/local/bin/thermos - upstart_update aurora-thermos-observer + sudo systemctl restart aurora-executor } function build_all { http://git-wip-us.apache.org/repos/asf/aurora/blob/c52137e2/examples/vagrant/provision-dev-cluster.sh ---------------------------------------------------------------------- diff --git a/examples/vagrant/provision-dev-cluster.sh b/examples/vagrant/provision-dev-cluster.sh index 66b99cf..4a02390 100755 --- a/examples/vagrant/provision-dev-cluster.sh +++ b/examples/vagrant/provision-dev-cluster.sh @@ -54,25 +54,27 @@ EOF function docker_setup { gpasswd -a vagrant docker echo 'DOCKER_OPTS="--storage-driver=aufs"' | sudo tee --append /etc/default/docker - service docker restart + systemctl restart docker } function start_services { # Executing true on failure to please bash -e in case services are already running - start zookeeper || true + systemctl start zookeeper || true } function prepare_sources { + apt-get install # Assign mesos command line arguments. cp /vagrant/examples/vagrant/mesos_config/etc_mesos-slave/* /etc/mesos-slave cp /vagrant/examples/vagrant/mesos_config/etc_mesos-master/* /etc/mesos-master - stop mesos-master || true - stop mesos-slave || true + systemctl stop mesos-master || true + systemctl stop mesos-slave || true # Remove slave metadata to ensure slave start does not pick up old state. rm -rf /var/lib/mesos/meta/slaves/latest - start mesos-master - start mesos-slave + systemctl start mesos-master + systemctl start mesos-slave + sudo cp /vagrant/examples/vagrant/systemd/*.service /etc/systemd/system cat > /usr/local/bin/update-sources <<EOF #!/bin/bash rsync -urzvhl /vagrant/ /home/vagrant/aurora \ @@ -81,7 +83,7 @@ rsync -urzvhl /vagrant/ /home/vagrant/aurora \ --exclude=/third_party \ --delete # Install/update the upstart configurations. -sudo cp /vagrant/examples/vagrant/upstart/*.conf /etc/init +sudo cp /vagrant/examples/vagrant/systemd/*.service /etc/systemd/system EOF chmod +x /usr/local/bin/update-sources update-sources > /dev/null http://git-wip-us.apache.org/repos/asf/aurora/blob/c52137e2/examples/vagrant/systemd/aurora-executor.service ---------------------------------------------------------------------- diff --git a/examples/vagrant/systemd/aurora-executor.service b/examples/vagrant/systemd/aurora-executor.service new file mode 100644 index 0000000..5a1a908 --- /dev/null +++ b/examples/vagrant/systemd/aurora-executor.service @@ -0,0 +1,31 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +[Unit] +Description=Aurora Thermos Observer +After=network.target +Wants=network.target + +[Service] +ExecStart=/home/vagrant/aurora/dist/thermos_observer.pex \ + --ip=192.168.33.7 \ + --port=1338 \ + --log_to_disk=NONE \ + --log_to_stderr=google:INFO +User=root +Group=root +Restart=always +RestartSec=20 +LimitNOFILE=16384 + +[Install] +WantedBy=multi-user.target http://git-wip-us.apache.org/repos/asf/aurora/blob/c52137e2/examples/vagrant/systemd/aurora-scheduler-kerberos.service ---------------------------------------------------------------------- diff --git a/examples/vagrant/systemd/aurora-scheduler-kerberos.service b/examples/vagrant/systemd/aurora-scheduler-kerberos.service new file mode 100644 index 0000000..1c2cfc4 --- /dev/null +++ b/examples/vagrant/systemd/aurora-scheduler-kerberos.service @@ -0,0 +1,64 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +[Unit] +Description=Aurora Scheduler (Kerberos e2e test) +After=network.target +Wants=network.target + +[Service] +# Environment variables control the behavior of the Mesos scheduler driver (libmesos). +Environment="GLOG_v=0" +Environment="LIBPROCESS_PORT=8083" +Environment="LIBPROCESS_IP=192.168.33.7" +Environment="MESOS_NATIVE_JAVA_LIBRARY=/usr/lib/libmesos.so" + +Environment="DIST_DIR=/home/vagrant/aurora/dist" + +# Flags that control the behavior of the JVM. +Environment="JAVA_OPTS='-Djava.library.path=/usr/lib \ + -Xdebug \ + -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=5005 \ + -Djava.security.krb5.conf=/home/vagrant/src/krb5-1.13.1/build/testdir/krb5.conf \ + -Dsun.security.krb5.debug=true \ + -Dsun.security.jgss.debug=true \ + -Djavax.net.debug=all'" +WorkingDirectory=/home/vagrant/aurora/dist/install/aurora-scheduler +ExecStart=/home/vagrant/aurora/dist/install/aurora-scheduler/bin/aurora-scheduler \ + -cluster_name=example \ + -http_port=8081 \ + -hostname=aurora.local \ + -native_log_quorum_size=1 \ + -zk_endpoints=localhost:2181 \ + -mesos_master_address=zk://localhost:2181/mesos \ + -serverset_path=/aurora/scheduler \ + -native_log_zk_group_path=/aurora/replicated-log \ + -native_log_file_path=/var/db/aurora \ + -backup_dir=/var/lib/aurora/backups \ + -thermos_executor_path=${DIST_DIR}/thermos_executor.pex \ + -thermos_executor_flags="--announcer-ensemble localhost:2181" \ + -allowed_container_types=MESOS,DOCKER \ + -shiro_realm_modules=KERBEROS5_AUTHN,INI_AUTHNZ \ + -shiro_ini_path=/home/vagrant/aurora/src/test/resources/org/apache/aurora/scheduler/http/api/security/shiro-example.ini \ + -http_authentication_mechanism=NEGOTIATE \ + -kerberos_server_keytab=/home/vagrant/krb5-1.13.1/build/testdir/HTTP-aurora.local.keytab \ + -kerberos_server_principal=HTTP/aurora.lo...@krbtest.com \ + -tier_config=/home/vagrant/aurora/src/main/resources/org/apache/aurora/scheduler/tiers.json \ + -offer_filter_duration=0secs +User=root +Group=root +Restart=always +RestartSec=20 +LimitNOFILE=16384 + +[Install] +WantedBy=multi-user.target http://git-wip-us.apache.org/repos/asf/aurora/blob/c52137e2/examples/vagrant/systemd/aurora-scheduler.service ---------------------------------------------------------------------- diff --git a/examples/vagrant/systemd/aurora-scheduler.service b/examples/vagrant/systemd/aurora-scheduler.service new file mode 100644 index 0000000..524cbb3 --- /dev/null +++ b/examples/vagrant/systemd/aurora-scheduler.service @@ -0,0 +1,67 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +[Unit] +Description=Aurora Scheduler +After=network.target +Wants=network.target + +[Service] +# Environment variables control the behavior of the Mesos scheduler driver (libmesos). +Environment="GLOG_v=1" +Environment="LIBPROCESS_PORT=8083" +Environment="LIBPROCESS_IP=192.168.33.7" +Environment="MESOS_NATIVE_JAVA_LIBRARY=/usr/lib/libmesos.so" + +Environment="DIST_DIR=/home/vagrant/aurora/dist" + +# Flags that control the behavior of the JVM. +Environment="JAVA_OPTS='-Djava.library.path=/usr/lib \ + -Xdebug \ + -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=5005'" +WorkingDirectory=/home/vagrant/aurora/dist/install/aurora-scheduler +ExecStart=/home/vagrant/aurora/dist/install/aurora-scheduler/bin/aurora-scheduler \ + -cluster_name=devcluster \ + -hostname=aurora.local \ + -http_port=8081 \ + -native_log_quorum_size=1 \ + -zk_endpoints=localhost:2181 \ + -mesos_master_address=zk://localhost:2181/mesos \ + -serverset_path=/aurora/scheduler \ + -native_log_zk_group_path=/aurora/replicated-log \ + -native_log_file_path=/var/db/aurora \ + -backup_dir=/var/lib/aurora/backups \ + -thermos_executor_path=${DIST_DIR}/thermos_executor.pex \ + -global_container_mounts=/home/vagrant/aurora/examples/vagrant/config:/home/vagrant/aurora/examples/vagrant/config:ro,/home/vagrant/aurora/.auroraversion:/home/vagrant/aurora/.auroraversion:ro \ + -thermos_executor_flags="--announcer-ensemble localhost:2181 --announcer-zookeeper-auth-config /home/vagrant/aurora/examples/vagrant/config/announcer-auth.json --mesos-containerizer-path=/usr/libexec/mesos/mesos-containerizer" \ + -allowed_container_types=MESOS,DOCKER \ + -http_authentication_mechanism=BASIC \ + -shiro_ini_path=etc/shiro.example.ini \ + -tier_config=/home/vagrant/aurora/src/main/resources/org/apache/aurora/scheduler/tiers.json \ + -mesos_role=aurora-role \ + -populate_discovery_info=true \ + -receive_revocable_resources=true \ + -enable_revocable_ram=true \ + -partition_aware=true \ + -allow_gpu_resource=true \ + -allow_container_volumes=true \ + -offer_filter_duration=0secs \ + -mesos_driver=V1_DRIVER \ + -unavailability_threshold=1mins +User=root +Group=root +Restart=always +RestartSec=20 +LimitNOFILE=16384 + +[Install] +WantedBy=multi-user.target http://git-wip-us.apache.org/repos/asf/aurora/blob/c52137e2/examples/vagrant/upstart/aurora-scheduler-kerberos.conf ---------------------------------------------------------------------- diff --git a/examples/vagrant/upstart/aurora-scheduler-kerberos.conf b/examples/vagrant/upstart/aurora-scheduler-kerberos.conf deleted file mode 100644 index 26b0241..0000000 --- a/examples/vagrant/upstart/aurora-scheduler-kerberos.conf +++ /dev/null @@ -1,56 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -description "aurora scheduler (kerberos e2e test)" -start on stopped rc RUNLEVEL=[2345] -respawn -post-stop exec sleep 5 -manual - -# Environment variables control the behavior of the Mesos scheduler driver (libmesos). -env GLOG_v=0 -env LIBPROCESS_PORT=8083 -env LIBPROCESS_IP=192.168.33.7 -env DIST_DIR=/home/vagrant/aurora/dist - -# Flags that control the behavior of the JVM. -env JAVA_OPTS='-Djava.library.path=/usr/lib - -Xdebug - -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=5005 - -Djava.security.krb5.conf=/home/vagrant/src/krb5-1.13.1/build/testdir/krb5.conf - -Dsun.security.krb5.debug=true - -Dsun.security.jgss.debug=true - -Djavax.net.debug=all -' - -chdir /home/vagrant/aurora/dist/install/aurora-scheduler -exec bin/aurora-scheduler \ - -cluster_name=example \ - -http_port=8081 \ - -hostname=aurora.local \ - -native_log_quorum_size=1 \ - -zk_endpoints=localhost:2181 \ - -mesos_master_address=zk://localhost:2181/mesos \ - -serverset_path=/aurora/scheduler \ - -native_log_zk_group_path=/aurora/replicated-log \ - -native_log_file_path=/var/db/aurora \ - -backup_dir=/var/lib/aurora/backups \ - -thermos_executor_path=$DIST_DIR/thermos_executor.pex \ - -thermos_executor_flags="--announcer-ensemble localhost:2181" \ - -allowed_container_types=MESOS,DOCKER \ - -shiro_realm_modules=KERBEROS5_AUTHN,INI_AUTHNZ \ - -shiro_ini_path=/home/vagrant/aurora/src/test/resources/org/apache/aurora/scheduler/http/api/security/shiro-example.ini \ - -http_authentication_mechanism=NEGOTIATE \ - -kerberos_server_keytab=/home/vagrant/krb5-1.13.1/build/testdir/HTTP-aurora.local.keytab \ - -kerberos_server_principal=HTTP/aurora.lo...@krbtest.com \ - -tier_config=/home/vagrant/aurora/src/main/resources/org/apache/aurora/scheduler/tiers.json \ - -offer_filter_duration=0secs http://git-wip-us.apache.org/repos/asf/aurora/blob/c52137e2/examples/vagrant/upstart/aurora-scheduler.conf ---------------------------------------------------------------------- diff --git a/examples/vagrant/upstart/aurora-scheduler.conf b/examples/vagrant/upstart/aurora-scheduler.conf deleted file mode 100644 index dbbe1d1..0000000 --- a/examples/vagrant/upstart/aurora-scheduler.conf +++ /dev/null @@ -1,58 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -description "aurora scheduler" -start on stopped rc RUNLEVEL=[2345] -respawn -post-stop exec sleep 5 - -# Environment variables control the behavior of the Mesos scheduler driver (libmesos). -env GLOG_v=1 -env LIBPROCESS_PORT=8083 -env LIBPROCESS_IP=192.168.33.7 -env DIST_DIR=/home/vagrant/aurora/dist - -# Flags that control the behavior of the JVM. -env JAVA_OPTS='-Djava.library.path=/usr/lib - -Xdebug - -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=5005 -' - -chdir /home/vagrant/aurora/dist/install/aurora-scheduler -exec bin/aurora-scheduler \ - -cluster_name=devcluster \ - -hostname=aurora.local \ - -http_port=8081 \ - -native_log_quorum_size=1 \ - -zk_endpoints=localhost:2181 \ - -mesos_master_address=zk://localhost:2181/mesos \ - -serverset_path=/aurora/scheduler \ - -native_log_zk_group_path=/aurora/replicated-log \ - -native_log_file_path=/var/db/aurora \ - -backup_dir=/var/lib/aurora/backups \ - -thermos_executor_path=$DIST_DIR/thermos_executor.pex \ - -global_container_mounts=/home/vagrant/aurora/examples/vagrant/config:/home/vagrant/aurora/examples/vagrant/config:ro,/home/vagrant/aurora/.auroraversion:/home/vagrant/aurora/.auroraversion:ro \ - -thermos_executor_flags="--announcer-ensemble localhost:2181 --announcer-zookeeper-auth-config /home/vagrant/aurora/examples/vagrant/config/announcer-auth.json --mesos-containerizer-path=/usr/libexec/mesos/mesos-containerizer" \ - -allowed_container_types=MESOS,DOCKER \ - -http_authentication_mechanism=BASIC \ - -shiro_ini_path=etc/shiro.example.ini \ - -tier_config=/home/vagrant/aurora/src/main/resources/org/apache/aurora/scheduler/tiers.json \ - -mesos_role=aurora-role \ - -populate_discovery_info=true \ - -receive_revocable_resources=true \ - -enable_revocable_ram=true \ - -partition_aware=true \ - -allow_gpu_resource=true \ - -allow_container_volumes=true \ - -offer_filter_duration=0secs \ - -mesos_driver=V1_DRIVER \ - -unavailability_threshold=1mins http://git-wip-us.apache.org/repos/asf/aurora/blob/c52137e2/examples/vagrant/upstart/aurora-thermos-observer.conf ---------------------------------------------------------------------- diff --git a/examples/vagrant/upstart/aurora-thermos-observer.conf b/examples/vagrant/upstart/aurora-thermos-observer.conf deleted file mode 100644 index dfb93b8..0000000 --- a/examples/vagrant/upstart/aurora-thermos-observer.conf +++ /dev/null @@ -1,22 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -description "aurora thermos observer" -start on stopped rc RUNLEVEL=[2345] -respawn -post-stop exec sleep 5 - -exec /home/vagrant/aurora/dist/thermos_observer.pex \ - --ip=192.168.33.7 \ - --port=1338 \ - --log_to_disk=NONE \ - --log_to_stderr=google:INFO http://git-wip-us.apache.org/repos/asf/aurora/blob/c52137e2/src/test/sh/org/apache/aurora/e2e/Dockerfile.netcat ---------------------------------------------------------------------- diff --git a/src/test/sh/org/apache/aurora/e2e/Dockerfile.netcat b/src/test/sh/org/apache/aurora/e2e/Dockerfile.netcat index 17a028e..a87df85 100644 --- a/src/test/sh/org/apache/aurora/e2e/Dockerfile.netcat +++ b/src/test/sh/org/apache/aurora/e2e/Dockerfile.netcat @@ -12,7 +12,7 @@ # limitations under the License. # -FROM buildpack-deps:jessie +FROM buildpack-deps:stretch RUN apt-get update && apt-get install -y netcat-openbsd COPY run-server.sh /usr/local/bin http://git-wip-us.apache.org/repos/asf/aurora/blob/c52137e2/src/test/sh/org/apache/aurora/e2e/Dockerfile.python ---------------------------------------------------------------------- diff --git a/src/test/sh/org/apache/aurora/e2e/Dockerfile.python b/src/test/sh/org/apache/aurora/e2e/Dockerfile.python index 96be893..86be6c0 100644 --- a/src/test/sh/org/apache/aurora/e2e/Dockerfile.python +++ b/src/test/sh/org/apache/aurora/e2e/Dockerfile.python @@ -12,6 +12,6 @@ # limitations under the License. # -FROM python:2.7-slim +FROM python:2.7-slim-stretch COPY http_example.py /tmp/ http://git-wip-us.apache.org/repos/asf/aurora/blob/c52137e2/src/test/sh/org/apache/aurora/e2e/http/http_example.aurora ---------------------------------------------------------------------- diff --git a/src/test/sh/org/apache/aurora/e2e/http/http_example.aurora b/src/test/sh/org/apache/aurora/e2e/http/http_example.aurora index 0361b36..8878ab9 100644 --- a/src/test/sh/org/apache/aurora/e2e/http/http_example.aurora +++ b/src/test/sh/org/apache/aurora/e2e/http/http_example.aurora @@ -129,7 +129,7 @@ job = Service( volumes = [ Volume(container_path="etc/rsyslog.d.container", host_path="/etc/rsyslog.d", mode=Mode("RO")), - Volume(container_path="etc/w3m.container", host_path="/etc/w3m", mode=Mode("RO")) + Volume(container_path="etc/w3m.container", host_path="/etc/X11", mode=Mode("RO")) ] jobs = [ http://git-wip-us.apache.org/repos/asf/aurora/blob/c52137e2/src/test/sh/org/apache/aurora/e2e/test_end_to_end.sh ---------------------------------------------------------------------- diff --git a/src/test/sh/org/apache/aurora/e2e/test_end_to_end.sh b/src/test/sh/org/apache/aurora/e2e/test_end_to_end.sh index 8f9a77f..888efe4 100755 --- a/src/test/sh/org/apache/aurora/e2e/test_end_to_end.sh +++ b/src/test/sh/org/apache/aurora/e2e/test_end_to_end.sh @@ -261,7 +261,7 @@ test_update() { local _update_id=$(aurora update list $_jobkey --status ROLLING_FORWARD \ | tail -n +2 | awk '{print $2}') aurora_admin scheduler_snapshot devcluster - sudo restart aurora-scheduler + sudo systemctl restart aurora-scheduler assert_update_state $_jobkey 'ROLLING_FORWARD' aurora update pause $_jobkey --message='hello' assert_update_state $_jobkey 'ROLL_FORWARD_PAUSED' @@ -313,7 +313,7 @@ test_partition_awareness() { aurora update start --wait $_delay_jobkey $_config # partition the agent - sudo stop mesos-slave + sudo systemctl stop mesos-slave # the default job should become LOST and then transition to PENDING wait_until_task_status $_default_jobkey "0" "PENDING" @@ -323,7 +323,7 @@ test_partition_awareness() { assert_task_status $_delay_jobkey "0" "PARTITIONED" # start the agent back up - sudo start mesos-slave + sudo systemctl start mesos-slave # This can be removed when https://issues.apache.org/jira/browse/MESOS-6406 is resolved. # We have to pause and let the agent reregister with Mesos, then ask Aurora to explicitly @@ -511,7 +511,7 @@ test_recovery_tool() { # Take a backup aurora_admin scheduler_backup_now $_cluster - sudo stop aurora-scheduler + sudo systemctl stop aurora-scheduler # Reset storage sudo rm -r $REPLICATED_LOG_DIR @@ -528,7 +528,7 @@ test_recovery_tool() { -native_log_zk_group_path=/aurora/replicated-log \ -native_log_file_path=$REPLICATED_LOG_DIR \ -zk_endpoints=localhost:2181 - sudo start aurora-scheduler + sudo systemctl start aurora-scheduler # This command exits non-zero if the update is not found. aurora update info $update @@ -657,14 +657,14 @@ setup_image_stores() { # build the appc image from the docker image docker2aci http_example_netcat-latest.tar - APPC_IMAGE_ID="sha512-$(sha512sum http_example_netcat-latest.aci | awk '{print $1}')" + APPC_IMAGE_ID="sha512-$(sha512sum library-http_example_netcat-latest.aci | awk '{print $1}')" export APPC_IMAGE_ID APPC_IMAGE_DIRECTORY="/tmp/mesos/images/appc/images/$APPC_IMAGE_ID" sudo mkdir -p "$APPC_IMAGE_DIRECTORY" - sudo tar -xf http_example_netcat-latest.aci -C "$APPC_IMAGE_DIRECTORY" + sudo tar -xf library-http_example_netcat-latest.aci -C "$APPC_IMAGE_DIRECTORY" # This restart is necessary for mesos to pick up the image from the local store. - sudo restart mesos-slave + sudo systemctl restart mesos-slave popd rm -rf "$TEMP_PATH"