This is an automated email from the ASF dual-hosted git repository.
marcoabreu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-mxnet.git
The following commit(s) were added to refs/heads/master by this push:
new b78265b [MXNET-793] Virtual testing with Qemu, refinement and extract
test results to root MXNet folder (#13065)
b78265b is described below
commit b78265bac0dc47c39085184023174be1d40fb172
Author: Pedro Larroy <[email protected]>
AuthorDate: Fri Nov 2 17:53:46 2018 +0100
[MXNET-793] Virtual testing with Qemu, refinement and extract test results
to root MXNet folder (#13065)
* Improve Qemu infrastructure
Add documentation about running it interactively
* Separate provision
* Improve provisioning
* Refine provisioning and interactive
* Cant provision when the volumes arent mounted
* Fix running tests
* raise log output to INFO
* adjust logging
* flush stdout and stderr
* Refine by copying test results back to the host
* Fix license
* remove config file and different way to run QEMU
* remove config file and different way to run QEMU, remove ansible
---
ci/README.md | 20 +++++++++-
ci/docker/Dockerfile.build.test.arm_qemu | 6 ++-
ci/docker/install/ubuntu_arm_qemu.sh | 5 ++-
ci/docker/qemu/ansible.cfg | 20 ----------
ci/docker/qemu/playbook.yml | 48 ------------------------
ci/docker/qemu/qemu_run.sh | 31 ----------------
ci/docker/qemu/runtime_functions.py | 35 +++++++++++++-----
ci/docker/qemu/vmcontrol.py | 63 +++++++++++++++++++++++++++-----
8 files changed, 104 insertions(+), 124 deletions(-)
diff --git a/ci/README.md b/ci/README.md
index 3737fc7..6c8a23f 100644
--- a/ci/README.md
+++ b/ci/README.md
@@ -98,5 +98,21 @@ To run the unit tests under qemu:
./build.py -p armv7 && ./build.py -p test.arm_qemu ./runtime_functions.py
run_ut_py3_qemu
```
-To get a shell on the container and debug issues with the emulator itself:
-Run the output of `./build.py -p test.arm_qemu --print-docker-run`
+To get a shell on the container and debug issues with the emulator itself, we
build the container
+and then execute it interactively. We can afterwards use port 2222 on the host
to connect with SSH.
+
+
+```
+ci/build.py -p test.arm_qemu -b && docker run -p2222:2222 -ti
mxnetci/build.test.arm_qemu
+```
+
+Then from another terminal:
+
+```
+ssh -o StrictHostKeyChecking=no -p 2222 qemu@localhost
+```
+
+There are two pre-configured users: `root` and `qemu` both without passwords.
+
+
+
diff --git a/ci/docker/Dockerfile.build.test.arm_qemu
b/ci/docker/Dockerfile.build.test.arm_qemu
index fde105c..68891a7 100644
--- a/ci/docker/Dockerfile.build.test.arm_qemu
+++ b/ci/docker/Dockerfile.build.test.arm_qemu
@@ -39,6 +39,8 @@ RUN /work/ubuntu_adduser.sh
COPY runtime_functions.sh /work/
COPY qemu/* /work/
-COPY qemu/ansible.cfg /etc/ansible/ansible.cfg
-CMD ["./runtime_functions.py","run_ut_py3_qemu"]
+# SSH to the Qemu VM
+EXPOSE 2222/tcp
+
+CMD ["./runtime_functions.py","run_qemu_interactive"]
diff --git a/ci/docker/install/ubuntu_arm_qemu.sh
b/ci/docker/install/ubuntu_arm_qemu.sh
index c30dc4f..79ab67b 100755
--- a/ci/docker/install/ubuntu_arm_qemu.sh
+++ b/ci/docker/install/ubuntu_arm_qemu.sh
@@ -31,6 +31,7 @@ apt-get install -y \
qemu-system-arm \
unzip \
bzip2 \
- vim-nox
+ vim-nox \
+ toilet
-pip3 install ansible ipython
+pip3 install ipython
diff --git a/ci/docker/qemu/ansible.cfg b/ci/docker/qemu/ansible.cfg
deleted file mode 100644
index 24e2ec8..0000000
--- a/ci/docker/qemu/ansible.cfg
+++ /dev/null
@@ -1,20 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-[defaults]
-host_key_checking = False
-ansible_python_interpreter = /usr/bin/python3
diff --git a/ci/docker/qemu/playbook.yml b/ci/docker/qemu/playbook.yml
deleted file mode 100644
index 3b9e7c5..0000000
--- a/ci/docker/qemu/playbook.yml
+++ /dev/null
@@ -1,48 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
----
-- name: provision QEMU VM
- hosts: all
- gather_facts: no
- become: true
- become_user: root
- tasks:
- - name: Wait until ssh is available
- wait_for_connection:
- delay: 0
- sleep: 3
- timeout: 400
- - command: hostname
- register: h
- - debug: msg="{{ h.stdout }}"
-
- - name: copy mxnet artifacts
- copy:
- src: "{{ item }}"
- dest: mxnet_dist/
- with_fileglob: "/work/mxnet/build/*.whl"
-
- - name: copy runtime_functions.py
- copy:
- src: "/work/runtime_functions.py"
- dest: .
- - file:
- path: runtime_functions.py
- mode: 0755
-
-
diff --git a/ci/docker/qemu/qemu_run.sh b/ci/docker/qemu/qemu_run.sh
deleted file mode 100755
index 53a6487..0000000
--- a/ci/docker/qemu/qemu_run.sh
+++ /dev/null
@@ -1,31 +0,0 @@
-#!/bin/bash
-
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-set -exuo pipefail
-
-qemu-system-arm -M virt -m 1024 \
- -kernel vmlinuz \
- -initrd initrd.img \
- -append 'root=/dev/vda1' \
- -drive if=none,file=vda.qcow2,format=qcow2,id=hd \
- -device virtio-blk-device,drive=hd \
- -netdev user,id=mynet,hostfwd=tcp::2222-:22 \
- -device virtio-net-device,netdev=mynet \
- -nographic \
- -display none
diff --git a/ci/docker/qemu/runtime_functions.py
b/ci/docker/qemu/runtime_functions.py
index 6cf01b6..5659775 100755
--- a/ci/docker/qemu/runtime_functions.py
+++ b/ci/docker/qemu/runtime_functions.py
@@ -33,6 +33,8 @@ import time
import sys
import types
import glob
+import vmcontrol
+from vmcontrol import qemu_ssh, qemu_provision, qemu_rsync_to_host, VM
def activate_this(base):
import site
@@ -53,21 +55,24 @@ def activate_this(base):
sys.path.remove(item)
sys.path[:0] = new_sys_path
+
+
+
def run_ut_py3_qemu():
+ """Run unit tests in the emulator and copy the results back to the host
through the mounted
+ volume in /mxnet"""
from vmcontrol import VM
with VM() as vm:
- logging.info("VM provisioning with ansible")
- check_call(["ansible-playbook", "-v", "-u", "qemu", "-i",
"localhost:{},".format(vm.ssh_port), "playbook.yml"])
- logging.info("VM provisioned successfully.")
- logging.info("sync tests")
- check_call(['rsync', '-e', 'ssh -p{}'.format(vm.ssh_port), '-a',
'mxnet/tests', 'qemu@localhost:mxnet'])
+ qemu_provision(vm.ssh_port)
logging.info("execute tests")
- check_call(["ssh", "-o", "ServerAliveInterval=5",
"-p{}".format(vm.ssh_port), "qemu@localhost", "./runtime_functions.py",
"run_ut_python3_qemu_internal"])
+ qemu_ssh(vm.ssh_port, "./runtime_functions.py",
"run_ut_python3_qemu_internal")
+ qemu_rsync_to_host(vm.ssh_port, "*.xml", "mxnet")
+ logging.info("copied to host")
logging.info("tests finished, vm shutdown.")
vm.shutdown()
def run_ut_python3_qemu_internal():
- """this runs inside the vm, it's run by the playbook above by ansible"""
+ """this runs inside the vm"""
pkg = glob.glob('mxnet_dist/*.whl')[0]
logging.info("=== NOW Running inside QEMU ===")
logging.info("PIP Installing %s", pkg)
@@ -75,8 +80,20 @@ def run_ut_python3_qemu_internal():
logging.info("PIP Installing mxnet/tests/requirements.txt")
check_call(['sudo', 'pip3', 'install', '-r',
'mxnet/tests/requirements.txt'])
logging.info("Running tests in mxnet/tests/python/unittest/")
- check_call(['nosetests', '--with-timer', '--with-xunit', '--xunit-file',
'nosetests_unittest.xml', '--verbose',
'mxnet/tests/python/unittest/test_ndarray.py:test_ndarray_fluent'])
+ check_call(['nosetests', '--with-timer', '--with-xunit', '--xunit-file',
'nosetests_unittest.xml', '--verbose', 'mxnet/tests/python/unittest/'])
+ # Example to run a single unit test:
+ # check_call(['nosetests', '--with-timer', '--with-xunit', '--xunit-file',
'nosetests_unittest.xml', '--verbose',
'mxnet/tests/python/unittest/test_ndarray.py:test_ndarray_fluent'])
+
+
+
+def run_qemu_interactive():
+ vm = VM(interactive=True)
+ vm.detach()
+ vm.start()
+ vm.wait()
+ logging.info("QEMU finished")
+################################
def parsed_args():
parser = argparse.ArgumentParser(description="""python runtime
functions""", epilog="")
@@ -95,7 +112,7 @@ def chdir_to_script_directory():
os.chdir(base)
def main():
- logging.getLogger().setLevel(logging.DEBUG)
+ logging.getLogger().setLevel(logging.INFO)
logging.basicConfig(format='{}: %(asctime)-15s
%(message)s'.format(script_name()))
chdir_to_script_directory()
diff --git a/ci/docker/qemu/vmcontrol.py b/ci/docker/qemu/vmcontrol.py
index 2262bc7..a7e8c0f 100644
--- a/ci/docker/qemu/vmcontrol.py
+++ b/ci/docker/qemu/vmcontrol.py
@@ -42,6 +42,8 @@ import shlex
#
# The VMs are provisioned after boot, tests are run and then they are stopped
#
+QEMU_SSH_PORT=2222
+QEMU_RAM=4096
QEMU_RUN="""
qemu-system-arm -M virt -m {ram} \
@@ -55,17 +57,32 @@ qemu-system-arm -M virt -m {ram} \
-display none -nographic
"""
+QEMU_RUN_INTERACTIVE="""
+qemu-system-arm -M virt -m {ram} \
+ -kernel vmlinuz \
+ -initrd initrd.img \
+ -append 'root=/dev/vda1' \
+ -drive if=none,file=vda.qcow2,format=qcow2,id=hd \
+ -device virtio-blk-device,drive=hd \
+ -netdev user,id=mynet,hostfwd=tcp::{ssh_port}-:22 \
+ -device virtio-net-device,netdev=mynet \
+ -nographic
+"""
+
+
class VMError(RuntimeError):
pass
class VM:
"""Control of the virtual machine"""
- def __init__(self, ssh_port=2222):
+ def __init__(self, ssh_port=QEMU_SSH_PORT, ram=QEMU_RAM,
interactive=False):
self.log = logging.getLogger(VM.__name__)
self.ssh_port = ssh_port
self.timeout_s = 300
self.qemu_process = None
self._detach = False
+ self._interactive = interactive
+ self.ram = ram
def __enter__(self):
self.start()
@@ -77,13 +94,22 @@ class VM:
self.terminate()
def start(self):
- self.log.info("Starting VM, ssh port redirected to localhost:%s",
self.ssh_port)
+ sys.stderr.flush()
+ call(['toilet', '-f', 'smbraille', 'Starting QEMU'])
+ sys.stdout.flush()
+ self.log.info("Starting VM, ssh port redirected to localhost:%s
(inside docker, not exposed by default)", self.ssh_port)
if self.is_running():
raise VMError("VM is running, shutdown first")
- self.qemu_process = run_qemu(self.ssh_port)
+ if self._interactive:
+ self.qemu_process =
Popen(shlex.split(QEMU_RUN_INTERACTIVE.format(ssh_port=self.ssh_port,
ram=self.ram)))
+ return
+ else:
+ self.log.info("Starting in non-interactive mode. Terminal output
is disabled.")
+ self.qemu_process =
Popen(shlex.split(QEMU_RUN.format(ssh_port=self.ssh_port, ram=self.ram)),
stdout=DEVNULL, stdin=DEVNULL, stderr=PIPE)
def keep_waiting():
return self.is_running()
+ logging.info("waiting for ssh to be open in the VM (timeout
{}s)".format(self.timeout_s))
ssh_working = wait_ssh_open('127.0.0.1', self.ssh_port, keep_waiting,
self.timeout_s)
if not self.is_running():
@@ -140,11 +166,28 @@ class VM:
logging.info("VM destructor hit")
self.terminate()
-def run_qemu(ssh_port=2222):
- cmd = QEMU_RUN.format(ssh_port=ssh_port, ram=4096)
- logging.info("QEMU command: %s", cmd)
- qemu_process = Popen(shlex.split(cmd), stdout=DEVNULL, stdin=DEVNULL,
stderr=PIPE)
- return qemu_process
+
+def qemu_ssh(ssh_port=QEMU_SSH_PORT, *args):
+ check_call(["ssh", "-o", "ServerAliveInterval=5", "-o",
"StrictHostKeyChecking=no", "-p{}".format(ssh_port), "qemu@localhost", *args])
+
+
+def qemu_rsync(ssh_port, local_path, remote_path):
+ check_call(['rsync', '-e', 'ssh -o StrictHostKeyChecking=no
-p{}'.format(ssh_port), '-a', local_path,
'qemu@localhost:{}'.format(remote_path)])
+
+def qemu_rsync_to_host(ssh_port, remote_path, local_path):
+ check_call(['rsync', '-e', 'ssh -o StrictHostKeyChecking=no
-p{}'.format(ssh_port), '-va', 'qemu@localhost:{}'.format(remote_path),
local_path])
+
+def qemu_provision(ssh_port=QEMU_SSH_PORT):
+ import glob
+ logging.info("Provisioning the VM with artifacts and sources")
+
+ artifact = glob.glob('/work/mxnet/build/*.whl')
+ for x in artifact:
+ qemu_rsync(ssh_port, x, 'mxnet_dist/')
+ qemu_rsync(ssh_port, '/work/runtime_functions.py','')
+ qemu_rsync(ssh_port, '/work/vmcontrol.py','')
+ qemu_rsync(ssh_port, 'mxnet/tests', 'mxnet')
+ logging.info("Provisioning completed successfully.")
def wait_ssh_open(server, port, keep_waiting=None, timeout=None):
@@ -159,7 +202,7 @@ def wait_ssh_open(server, port, keep_waiting=None,
timeout=None):
import errno
import time
log = logging.getLogger('wait_ssh_open')
- sleep_s = 0
+ sleep_s = 1
if timeout:
from time import time as now
# time module is needed to calc timeout shared between two exceptions
@@ -183,7 +226,7 @@ def wait_ssh_open(server, port, keep_waiting=None,
timeout=None):
log.debug("connect timeout %d s", next_timeout)
s.settimeout(next_timeout)
- log.info("connect %s:%d", server, port)
+ log.debug("connect %s:%d", server, port)
s.connect((server, port))
ret = s.recv(1024).decode()
if ret and ret.startswith('SSH'):
