marcoabreu closed pull request #13065: [MXNET-793] Virtual testing with Qemu,
refinement and extract test results to root MXNet folder
URL: https://github.com/apache/incubator-mxnet/pull/13065
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/ci/README.md b/ci/README.md
index 3737fc74065..6c8a23f03cf 100644
--- a/ci/README.md
+++ b/ci/README.md
@@ -98,5 +98,21 @@ To run the unit tests under qemu:
./build.py -p armv7 && ./build.py -p test.arm_qemu ./runtime_functions.py
run_ut_py3_qemu
```
-To get a shell on the container and debug issues with the emulator itself:
-Run the output of `./build.py -p test.arm_qemu --print-docker-run`
+To get a shell on the container and debug issues with the emulator itself, we
build the container
+and then execute it interactively. We can afterwards use port 2222 on the host
to connect with SSH.
+
+
+```
+ci/build.py -p test.arm_qemu -b && docker run -p2222:2222 -ti
mxnetci/build.test.arm_qemu
+```
+
+Then from another terminal:
+
+```
+ssh -o StrictHostKeyChecking=no -p 2222 qemu@localhost
+```
+
+There are two pre-configured users: `root` and `qemu` both without passwords.
+
+
+
diff --git a/ci/docker/Dockerfile.build.test.arm_qemu
b/ci/docker/Dockerfile.build.test.arm_qemu
index fde105c3f98..68891a72c28 100644
--- a/ci/docker/Dockerfile.build.test.arm_qemu
+++ b/ci/docker/Dockerfile.build.test.arm_qemu
@@ -39,6 +39,8 @@ RUN /work/ubuntu_adduser.sh
COPY runtime_functions.sh /work/
COPY qemu/* /work/
-COPY qemu/ansible.cfg /etc/ansible/ansible.cfg
-CMD ["./runtime_functions.py","run_ut_py3_qemu"]
+# SSH to the Qemu VM
+EXPOSE 2222/tcp
+
+CMD ["./runtime_functions.py","run_qemu_interactive"]
diff --git a/ci/docker/install/ubuntu_arm_qemu.sh
b/ci/docker/install/ubuntu_arm_qemu.sh
index c30dc4f13d1..79ab67bfdbe 100755
--- a/ci/docker/install/ubuntu_arm_qemu.sh
+++ b/ci/docker/install/ubuntu_arm_qemu.sh
@@ -31,6 +31,7 @@ apt-get install -y \
qemu-system-arm \
unzip \
bzip2 \
- vim-nox
+ vim-nox \
+ toilet
-pip3 install ansible ipython
+pip3 install ipython
diff --git a/ci/docker/qemu/ansible.cfg b/ci/docker/qemu/ansible.cfg
deleted file mode 100644
index 24e2ec87b94..00000000000
--- a/ci/docker/qemu/ansible.cfg
+++ /dev/null
@@ -1,20 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-[defaults]
-host_key_checking = False
-ansible_python_interpreter = /usr/bin/python3
diff --git a/ci/docker/qemu/playbook.yml b/ci/docker/qemu/playbook.yml
deleted file mode 100644
index 3b9e7c52bde..00000000000
--- a/ci/docker/qemu/playbook.yml
+++ /dev/null
@@ -1,48 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
----
-- name: provision QEMU VM
- hosts: all
- gather_facts: no
- become: true
- become_user: root
- tasks:
- - name: Wait until ssh is available
- wait_for_connection:
- delay: 0
- sleep: 3
- timeout: 400
- - command: hostname
- register: h
- - debug: msg="{{ h.stdout }}"
-
- - name: copy mxnet artifacts
- copy:
- src: "{{ item }}"
- dest: mxnet_dist/
- with_fileglob: "/work/mxnet/build/*.whl"
-
- - name: copy runtime_functions.py
- copy:
- src: "/work/runtime_functions.py"
- dest: .
- - file:
- path: runtime_functions.py
- mode: 0755
-
-
diff --git a/ci/docker/qemu/qemu_run.sh b/ci/docker/qemu/qemu_run.sh
deleted file mode 100755
index 53a6487aa0d..00000000000
--- a/ci/docker/qemu/qemu_run.sh
+++ /dev/null
@@ -1,31 +0,0 @@
-#!/bin/bash
-
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-set -exuo pipefail
-
-qemu-system-arm -M virt -m 1024 \
- -kernel vmlinuz \
- -initrd initrd.img \
- -append 'root=/dev/vda1' \
- -drive if=none,file=vda.qcow2,format=qcow2,id=hd \
- -device virtio-blk-device,drive=hd \
- -netdev user,id=mynet,hostfwd=tcp::2222-:22 \
- -device virtio-net-device,netdev=mynet \
- -nographic \
- -display none
diff --git a/ci/docker/qemu/runtime_functions.py
b/ci/docker/qemu/runtime_functions.py
index 6cf01b6f912..5659775c8b8 100755
--- a/ci/docker/qemu/runtime_functions.py
+++ b/ci/docker/qemu/runtime_functions.py
@@ -33,6 +33,8 @@
import sys
import types
import glob
+import vmcontrol
+from vmcontrol import qemu_ssh, qemu_provision, qemu_rsync_to_host, VM
def activate_this(base):
import site
@@ -53,21 +55,24 @@ def activate_this(base):
sys.path.remove(item)
sys.path[:0] = new_sys_path
+
+
+
def run_ut_py3_qemu():
+ """Run unit tests in the emulator and copy the results back to the host
through the mounted
+ volume in /mxnet"""
from vmcontrol import VM
with VM() as vm:
- logging.info("VM provisioning with ansible")
- check_call(["ansible-playbook", "-v", "-u", "qemu", "-i",
"localhost:{},".format(vm.ssh_port), "playbook.yml"])
- logging.info("VM provisioned successfully.")
- logging.info("sync tests")
- check_call(['rsync', '-e', 'ssh -p{}'.format(vm.ssh_port), '-a',
'mxnet/tests', 'qemu@localhost:mxnet'])
+ qemu_provision(vm.ssh_port)
logging.info("execute tests")
- check_call(["ssh", "-o", "ServerAliveInterval=5",
"-p{}".format(vm.ssh_port), "qemu@localhost", "./runtime_functions.py",
"run_ut_python3_qemu_internal"])
+ qemu_ssh(vm.ssh_port, "./runtime_functions.py",
"run_ut_python3_qemu_internal")
+ qemu_rsync_to_host(vm.ssh_port, "*.xml", "mxnet")
+ logging.info("copied to host")
logging.info("tests finished, vm shutdown.")
vm.shutdown()
def run_ut_python3_qemu_internal():
- """this runs inside the vm, it's run by the playbook above by ansible"""
+ """this runs inside the vm"""
pkg = glob.glob('mxnet_dist/*.whl')[0]
logging.info("=== NOW Running inside QEMU ===")
logging.info("PIP Installing %s", pkg)
@@ -75,8 +80,20 @@ def run_ut_python3_qemu_internal():
logging.info("PIP Installing mxnet/tests/requirements.txt")
check_call(['sudo', 'pip3', 'install', '-r',
'mxnet/tests/requirements.txt'])
logging.info("Running tests in mxnet/tests/python/unittest/")
- check_call(['nosetests', '--with-timer', '--with-xunit', '--xunit-file',
'nosetests_unittest.xml', '--verbose',
'mxnet/tests/python/unittest/test_ndarray.py:test_ndarray_fluent'])
+ check_call(['nosetests', '--with-timer', '--with-xunit', '--xunit-file',
'nosetests_unittest.xml', '--verbose', 'mxnet/tests/python/unittest/'])
+ # Example to run a single unit test:
+ # check_call(['nosetests', '--with-timer', '--with-xunit', '--xunit-file',
'nosetests_unittest.xml', '--verbose',
'mxnet/tests/python/unittest/test_ndarray.py:test_ndarray_fluent'])
+
+
+
+def run_qemu_interactive():
+ vm = VM(interactive=True)
+ vm.detach()
+ vm.start()
+ vm.wait()
+ logging.info("QEMU finished")
+################################
def parsed_args():
parser = argparse.ArgumentParser(description="""python runtime
functions""", epilog="")
@@ -95,7 +112,7 @@ def chdir_to_script_directory():
os.chdir(base)
def main():
- logging.getLogger().setLevel(logging.DEBUG)
+ logging.getLogger().setLevel(logging.INFO)
logging.basicConfig(format='{}: %(asctime)-15s
%(message)s'.format(script_name()))
chdir_to_script_directory()
diff --git a/ci/docker/qemu/vmcontrol.py b/ci/docker/qemu/vmcontrol.py
index 2262bc77bf9..a7e8c0ff012 100644
--- a/ci/docker/qemu/vmcontrol.py
+++ b/ci/docker/qemu/vmcontrol.py
@@ -42,6 +42,8 @@
#
# The VMs are provisioned after boot, tests are run and then they are stopped
#
+QEMU_SSH_PORT=2222
+QEMU_RAM=4096
QEMU_RUN="""
qemu-system-arm -M virt -m {ram} \
@@ -55,17 +57,32 @@
-display none -nographic
"""
+QEMU_RUN_INTERACTIVE="""
+qemu-system-arm -M virt -m {ram} \
+ -kernel vmlinuz \
+ -initrd initrd.img \
+ -append 'root=/dev/vda1' \
+ -drive if=none,file=vda.qcow2,format=qcow2,id=hd \
+ -device virtio-blk-device,drive=hd \
+ -netdev user,id=mynet,hostfwd=tcp::{ssh_port}-:22 \
+ -device virtio-net-device,netdev=mynet \
+ -nographic
+"""
+
+
class VMError(RuntimeError):
pass
class VM:
"""Control of the virtual machine"""
- def __init__(self, ssh_port=2222):
+ def __init__(self, ssh_port=QEMU_SSH_PORT, ram=QEMU_RAM,
interactive=False):
self.log = logging.getLogger(VM.__name__)
self.ssh_port = ssh_port
self.timeout_s = 300
self.qemu_process = None
self._detach = False
+ self._interactive = interactive
+ self.ram = ram
def __enter__(self):
self.start()
@@ -77,13 +94,22 @@ def __exit__(self, exc_type, exc_value, traceback):
self.terminate()
def start(self):
- self.log.info("Starting VM, ssh port redirected to localhost:%s",
self.ssh_port)
+ sys.stderr.flush()
+ call(['toilet', '-f', 'smbraille', 'Starting QEMU'])
+ sys.stdout.flush()
+ self.log.info("Starting VM, ssh port redirected to localhost:%s
(inside docker, not exposed by default)", self.ssh_port)
if self.is_running():
raise VMError("VM is running, shutdown first")
- self.qemu_process = run_qemu(self.ssh_port)
+ if self._interactive:
+ self.qemu_process =
Popen(shlex.split(QEMU_RUN_INTERACTIVE.format(ssh_port=self.ssh_port,
ram=self.ram)))
+ return
+ else:
+ self.log.info("Starting in non-interactive mode. Terminal output
is disabled.")
+ self.qemu_process =
Popen(shlex.split(QEMU_RUN.format(ssh_port=self.ssh_port, ram=self.ram)),
stdout=DEVNULL, stdin=DEVNULL, stderr=PIPE)
def keep_waiting():
return self.is_running()
+ logging.info("waiting for ssh to be open in the VM (timeout
{}s)".format(self.timeout_s))
ssh_working = wait_ssh_open('127.0.0.1', self.ssh_port, keep_waiting,
self.timeout_s)
if not self.is_running():
@@ -140,11 +166,28 @@ def __del__(self):
logging.info("VM destructor hit")
self.terminate()
-def run_qemu(ssh_port=2222):
- cmd = QEMU_RUN.format(ssh_port=ssh_port, ram=4096)
- logging.info("QEMU command: %s", cmd)
- qemu_process = Popen(shlex.split(cmd), stdout=DEVNULL, stdin=DEVNULL,
stderr=PIPE)
- return qemu_process
+
+def qemu_ssh(ssh_port=QEMU_SSH_PORT, *args):
+ check_call(["ssh", "-o", "ServerAliveInterval=5", "-o",
"StrictHostKeyChecking=no", "-p{}".format(ssh_port), "qemu@localhost", *args])
+
+
+def qemu_rsync(ssh_port, local_path, remote_path):
+ check_call(['rsync', '-e', 'ssh -o StrictHostKeyChecking=no
-p{}'.format(ssh_port), '-a', local_path,
'qemu@localhost:{}'.format(remote_path)])
+
+def qemu_rsync_to_host(ssh_port, remote_path, local_path):
+ check_call(['rsync', '-e', 'ssh -o StrictHostKeyChecking=no
-p{}'.format(ssh_port), '-va', 'qemu@localhost:{}'.format(remote_path),
local_path])
+
+def qemu_provision(ssh_port=QEMU_SSH_PORT):
+ import glob
+ logging.info("Provisioning the VM with artifacts and sources")
+
+ artifact = glob.glob('/work/mxnet/build/*.whl')
+ for x in artifact:
+ qemu_rsync(ssh_port, x, 'mxnet_dist/')
+ qemu_rsync(ssh_port, '/work/runtime_functions.py','')
+ qemu_rsync(ssh_port, '/work/vmcontrol.py','')
+ qemu_rsync(ssh_port, 'mxnet/tests', 'mxnet')
+ logging.info("Provisioning completed successfully.")
def wait_ssh_open(server, port, keep_waiting=None, timeout=None):
@@ -159,7 +202,7 @@ def wait_ssh_open(server, port, keep_waiting=None,
timeout=None):
import errno
import time
log = logging.getLogger('wait_ssh_open')
- sleep_s = 0
+ sleep_s = 1
if timeout:
from time import time as now
# time module is needed to calc timeout shared between two exceptions
@@ -183,7 +226,7 @@ def wait_ssh_open(server, port, keep_waiting=None,
timeout=None):
log.debug("connect timeout %d s", next_timeout)
s.settimeout(next_timeout)
- log.info("connect %s:%d", server, port)
+ log.debug("connect %s:%d", server, port)
s.connect((server, port))
ret = s.recv(1024).decode()
if ret and ret.startswith('SSH'):
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
