[osv-dev] [PATCH 3/3] externals: eliminate aarch64 gcc and boost libraries from externals

[Waldemar Kozaczuk]

This patch eliminates aarch64 gcc and boost libraries from externals
in lieu of new mechanism to download the packages from Fedora
repository.

Fixes #743

Signed-off-by: Waldemar Kozaczuk 
---
 .gitignore|  1 +
 .gitmodules   |  6 
 Makefile  | 60 +--
 external/aarch64/gcc.bin  |  1 -
 external/aarch64/misc.bin |  1 -
 5 files changed, 21 insertions(+), 48 deletions(-)
 delete mode 16 external/aarch64/gcc.bin
 delete mode 16 external/aarch64/misc.bin

diff --git a/.gitignore b/.gitignore
index 16e192db..398c5de7 100644
--- a/.gitignore
+++ b/.gitignore
@@ -49,3 +49,4 @@ modules/libyaml/usr.manifest
 modules/dl_tests/usr.manifest
 .idea
 compile_commands.json
+downloaded_packages
diff --git a/.gitmodules b/.gitmodules
index da6d5d62..b6ae6ac1 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -6,12 +6,6 @@
path = apps
url = ../../cloudius-systems/osv-apps
ignore = dirty
-[submodule "external/aarch64/gcc.bin"]
-   path = external/aarch64/gcc.bin
-   url = ../../cloudius-systems/aarch64-gcc.bin.git
-[submodule "external/aarch64/misc.bin"]
-   path = external/aarch64/misc.bin
-   url = ../../cloudius-systems/aarch64-misc.bin.git
 [submodule "modules/httpserver/swagger-ui"]
path = modules/httpserver-html5-gui/swagger-ui
url = ../../cloudius-systems/swagger-ui.git
diff --git a/Makefile b/Makefile
index 0e5d4d32..e8e110a5 100644
--- a/Makefile
+++ b/Makefile
@@ -205,34 +205,16 @@ cscope:
 
 ###
 
-
-# The user can override the build_env variable (or one or more of *_env
-# variables below) to decide if to take the host's C/C++ libraries, or
-# those from the external/ directory.
-build_env ?= $(if $(filter $(host_arch), $(arch)),host,external)
-ifeq ($(build_env), host)
-gcc_lib_env ?= host
-cxx_lib_env ?= host
-gcc_include_env ?= host
-boost_env ?= host
-else
-gcc_lib_env ?= external
-cxx_lib_env ?= external
-gcc_include_env ?= external
-boost_env ?= external
-endif
-
-
 local-includes =
 INCLUDES = $(local-includes) -Iarch/$(arch) -I. -Iinclude  -Iarch/common
 INCLUDES += -isystem include/glibc-compat
 
-gccbase = external/$(arch)/gcc.bin
-miscbase = external/$(arch)/misc.bin
+aarch64_gccbase = downloaded_packages/aarch64/gcc/install
+aarch64_boostbase = downloaded_packages/aarch64/boost/install
 
-ifeq ($(gcc_include_env), external)
-  gcc-inc-base := $(dir $(shell find $(gccbase)/ -name vector | grep -v -e 
debug/vector$$ -e profile/vector$$))
-  gcc-inc-base3 := $(dir $(shell dirname `find $(gccbase)/ -name c++config.h | 
grep -v /32/`))
+ifeq ($(arch),aarch64)
+  gcc-inc-base := $(dir $(shell find $(aarch64_gccbase)/ -name vector | grep 
-v -e debug/vector$$ -e profile/vector$$ -e experimental/vector$$))
+  gcc-inc-base3 := $(dir $(shell dirname `find $(aarch64_gccbase)/ -name 
c++config.h | grep -v /32/`))
   INCLUDES += -isystem $(gcc-inc-base)
   INCLUDES += -isystem $(gcc-inc-base3)
 endif
@@ -247,7 +229,7 @@ INCLUDES += -isystem $(libfdt_base)
 endif
 
 INCLUDES += $(boost-includes)
-ifeq ($(gcc_include_env), host)
+ifeq ($(arch),x64)
 # Starting in Gcc 6, the standard C++ header files (which we do not change)
 # must precede in the include path the C header files (which we replace).
 # This is explained in https://gcc.gnu.org/bugzilla/show_bug.cgi?id=70722.
@@ -257,8 +239,8 @@ INCLUDES += $(shell $(CXX) -E -xc++ - -v &1 | 
awk '/^End/ {exit} /^
 endif
 INCLUDES += -isystem include/api
 INCLUDES += -isystem include/api/$(arch)
-ifeq ($(gcc_include_env), external)
-  gcc-inc-base2 := $(dir $(shell find $(gccbase)/ -name unwind.h))
+ifeq ($(arch),aarch64)
+  gcc-inc-base2 := $(dir $(shell find $(aarch64_gccbase)/ -name unwind.h))
   # must be after include/api, since it includes some libc-style headers:
   INCLUDES += -isystem $(gcc-inc-base2)
 endif
@@ -289,7 +271,7 @@ $(out)/musl/%.o: source-dialects =
 
 kernel-defines = -D_KERNEL $(source-dialects)
 
-gcc-sysroot = $(if $(CROSS_PREFIX), --sysroot external/$(arch)/gcc.bin) \
+gcc-sysroot = $(if $(CROSS_PREFIX), --sysroot $(aarch64_gccbase)) \
 
 # This play the same role as "_KERNEL", but _KERNEL unfortunately is too
 # overloaded. A lot of files will expect it to be set no matter what, specially
@@ -314,11 +296,9 @@ COMMON = $(autodepend) -g -Wall -Wno-pointer-arith 
$(CFLAGS_WERROR) -Wformat=0 -
-include compiler/include/intrinsics.hh \
$(arch-cflags) $(conf-opt) $(acpi-defines) $(tracing-flags) 
$(gcc-sysroot) \
$(configuration) -D__OSV__ -D__XEN_INTERFACE_VERSION__="0x00030207" 
-DARCH_STRING=$(ARCH_STR) $(EXTRA_FLAGS)
-ifeq ($(gcc_include_env), external)
-ifeq ($(boost_env), external)
+ifeq ($(arch),aarch64)
   COMMON += -nostdinc
 endif
-endif
 
 tracing-flags-0 =
 tracing-flags-1 = -finstrument-functions 

[osv-dev] [PATCH 1/3] scripts: new script to download and extract arbitrary Fedora rpm

[Waldemar Kozaczuk]

Signed-off-by: Waldemar Kozaczuk 
---
 scripts/download_rpm_package.sh | 34 +
 1 file changed, 34 insertions(+)
 create mode 100755 scripts/download_rpm_package.sh

diff --git a/scripts/download_rpm_package.sh b/scripts/download_rpm_package.sh
new file mode 100755
index ..562b379a
--- /dev/null
+++ b/scripts/download_rpm_package.sh
@@ -0,0 +1,34 @@
+#!/bin/bash
+
+package="$1"
+release=$2
+out_dir=$3
+
+arch=aarch64
+
+letter=${package:0:1}
+
+main_repo_url="http://mirrors.kernel.org/fedora/releases/$release/Everything/$arch/os/Packages/$letter/;
+version=$(wget -t 1 -qO- $main_repo_url | grep "${package}-[0-9].*$arch" | 
grep -Po ">.*<" | sed -e "s/>${package}-\(.*\)\.$arch\.rpmhttp://archives.fedoraproject.org/pub/archive/fedora/linux/releases/$release/Everything/$arch/os/Packages/$letter/;
+if [[ "${version}" != "" ]]; then
+  file_name="${package}-${version}.aarch64.rpm"
+  full_url="${main_repo_url}${file_name}"
+else
+  version=$(wget -t 1 -qO- $archive_repo_url | grep "${package}-[0-9].*$arch" 
| grep -Po "" | sed -e "s/https://groups.google.com/d/msgid/osv-dev/20200225055252.6672-1-jwkozaczuk%40gmail.com.

[osv-dev] [PATCH 2/3] externals: enhance setup.py to download aarch64 gcc and boost code and libraries

[Waldemar Kozaczuk]

References #743

Signed-off-by: Waldemar Kozaczuk 
---
 scripts/setup.py | 40 +++-
 1 file changed, 39 insertions(+), 1 deletion(-)

diff --git a/scripts/setup.py b/scripts/setup.py
index db831f0f..946d1e4e 100755
--- a/scripts/setup.py
+++ b/scripts/setup.py
@@ -3,7 +3,7 @@
 # set up a development environment for OSv.  Run as root.
 
 import sys, argparse
-import subprocess
+import subprocess, os
 
 standard_ec2_packages = ['python-pip', 'wget']
 standard_ec2_post_install = ['pip install awscli &&'
@@ -64,6 +64,40 @@ class Fedora(object):
 test_packages = ['openssl-devel']
 ec2_post_install = standard_ec2_post_install
 
+def aarch64_download(self, version):
+gcc_packages = ['gcc',
+'glibc',
+'glibc-devel',
+'libgcc',
+'libstdc++',
+'libstdc++-devel',
+'libstdc++-static']
+boost_packages = ['boost-devel',
+  'boost-static',
+  'boost-system']
+osv_root = os.path.join(os.path.dirname(os.path.abspath(__file__)), 
'..')
+script_path = '%s/scripts/download_rpm_package.sh' % osv_root
+destination = '%s/downloaded_packages/aarch64' % osv_root
+##
+# The setup.py is typically run as root to allow yum properly install 
packages
+# This however would cause all files downloaded to 
downloaded_packages/aarch64 directory
+# get created and owned by the root user which in most cases is not 
desirable
+# To prevent that let us compare current process user id with the 
owner id of osv root
+# directory and if different run all download command with the same 
user as the one owning
+# the root directory
+current_user_id = os.getuid()
+osv_root_owner_id = os.stat(osv_root).st_uid
+if current_user_id != osv_root_owner_id and current_user_id == 0:
+command_prefix = "sudo -u '#%d'" % osv_root_owner_id # Most likely 
setup.py is run by root so let us use sudo
+else:
+command_prefix = ''
+
+install_commands = ['%s %s %s %s %s/gcc' % (command_prefix, 
script_path, package, version, destination) for package in gcc_packages]
+install_commands += ['%s %s %s %s %s/boost' % (command_prefix, 
script_path, package, version, destination) for package in boost_packages]
+install_commands = ['%s rm -rf %s/gcc/install' % (command_prefix, 
destination),
+'%s rm -rf %s/boost/install' % (command_prefix, 
destination)] + install_commands
+return ' && '.join(install_commands)
+
 class Fedora_25(object):
 packages = ['java-1.8.0-openjdk', 'python2-requests', 'openssl-devel', 
'lua-5.3.*', 'lua-devel-5.3.*']
 ec2_packages = []
@@ -361,6 +395,10 @@ for distro in distros:
 if cmdargs.test:
 pkg += distro.test_packages + dver.test_packages
 subprocess.check_call(distro.install + ' ' + str.join(' ', 
pkg), shell=True)
+if 'aarch64_download' in dir(distro):
+print('Downloading aarch64 packages to cross-compile ARM 
version ...')
+
subprocess.check_call(distro.aarch64_download(dver.version), shell=True)
+print('Downloaded all aarch64 packages!')
 if cmdargs.ec2:
 if distro.ec2_post_install:
 subprocess.check_call(distro.ec2_post_install, 
shell=True)
-- 
2.20.1

-- 
You received this message because you are subscribed to the Google Groups "OSv 
Development" group.
To unsubscribe from this group and stop receiving emails from it, send an email 
to osv-dev+unsubscr...@googlegroups.com.
To view this discussion on the web visit 
https://groups.google.com/d/msgid/osv-dev/20200225055252.6672-2-jwkozaczuk%40gmail.com.

[osv-dev] [COMMIT osv master] scripts/loader.py: make osv info callouts work with older C++ ABI

[Commit Bot]

From: Waldemar Kozaczuk 
Committer: Waldemar Kozaczuk 
Branch: master

scripts/loader.py: make osv info callouts work with older C++ ABI

Signed-off-by: Waldemar Kozaczuk 

---
diff --git a/scripts/loader.py b/scripts/loader.py
--- a/scripts/loader.py
+++ b/scripts/loader.py
@@ -1034,7 +1034,10 @@ def invoke(self, arg, for_tty):
 fname = callout['c_fn']
 
 # time
-t = int(callout['c_to_ns']['__d']['__r'])
+try:
+t = int(callout['c_to_ns']['__d']['__r'])
+except gdb.error:
+t = int(callout['c_to_ns'])
 
 # flags
 CALLOUT_ACTIVE = 0x0002

-- 
You received this message because you are subscribed to the Google Groups "OSv 
Development" group.
To unsubscribe from this group and stop receiving emails from it, send an email 
to osv-dev+unsubscr...@googlegroups.com.
To view this discussion on the web visit 
https://groups.google.com/d/msgid/osv-dev/11da50059f5fda49%40google.com.

Re: [osv-dev] Benchmarking OSv

[tweekley5]

As follows:
OSv booted on KVM-
1 core: 21.18s
2: 12.25
4: 7.08 
8: 4.95s
16: 6.59s
32: 9.28s

Now on just regular Ubuntu it is as follows:
1: 15.22s
2: 9.64s
4: 6.4s
8: 3.6s
16: 2.93s
32: 2.21s


To your networking statement, is there a way to disable setting up the 
networking environment? My benchmark does not require any form of an internet 
connection.

In the event anyone is curious as to what I am bench marking, it is the 
Splash-2 apps / kernels with a rather large problem size being solved. In this 
benchmark it is a 2048x2048 matrix 

-- 
You received this message because you are subscribed to the Google Groups "OSv 
Development" group.
To unsubscribe from this group and stop receiving emails from it, send an email 
to osv-dev+unsubscr...@googlegroups.com.
To view this discussion on the web visit 
https://groups.google.com/d/msgid/osv-dev/10cb923a-c7b3-476c-92cd-dd9a6db0c7b9%40googlegroups.com.

Re: [osv-dev] Benchmarking OSv

[Dor Laor]

It depends on many factors. You should check what's the bottleneck first.
I'd start with a single core and later with 2 cores and grow it exponentially.
It either may be the IO overhead (most chances the network
virtualization) or the
cpu overhead with regard to locking. OSv has a different filesystem too.
Let's see what you have to share first.

On Mon, Feb 24, 2020 at 6:32 PM  wrote:
>
> Running some benchmarks on OSv. Getting some results opposite to what I was 
> expecting based off what I have learned of unikernels. When running my 
> benchmark on the host with 32 cores, it runs in about 2s. However, when 
> running the OSv build inside on KVM (run.py), I pass in 32 cores and it takes 
> 9s to complete. My assumption was it would be the same speed, possibly 
> faster. Any ideas as to why the decrease?
>
> --
> You received this message because you are subscribed to the Google Groups 
> "OSv Development" group.
> To unsubscribe from this group and stop receiving emails from it, send an 
> email to osv-dev+unsubscr...@googlegroups.com.
> To view this discussion on the web visit 
> https://groups.google.com/d/msgid/osv-dev/cac7dca2-eaa3-4c66-a8b7-705182e67942%40googlegroups.com.

-- 
You received this message because you are subscribed to the Google Groups "OSv 
Development" group.
To unsubscribe from this group and stop receiving emails from it, send an email 
to osv-dev+unsubscr...@googlegroups.com.
To view this discussion on the web visit 
https://groups.google.com/d/msgid/osv-dev/CAKUaUn7NHQ1mFLQwANH6%2BF1NXx%2ByOzRQ8eYjN_GD3qTF9o2adQ%40mail.gmail.com.

[osv-dev] Benchmarking OSv

[tweekley5]

Running some benchmarks on OSv. Getting some results opposite to what I was 
expecting based off what I have learned of unikernels. When running my 
benchmark on the host with 32 cores, it runs in about 2s. However, when running 
the OSv build inside on KVM (run.py), I pass in 32 cores and it takes 9s to 
complete. My assumption was it would be the same speed, possibly faster. Any 
ideas as to why the decrease?

-- 
You received this message because you are subscribed to the Google Groups "OSv 
Development" group.
To unsubscribe from this group and stop receiving emails from it, send an email 
to osv-dev+unsubscr...@googlegroups.com.
To view this discussion on the web visit 
https://groups.google.com/d/msgid/osv-dev/cac7dca2-eaa3-4c66-a8b7-705182e67942%40googlegroups.com.

Re: [osv-dev] [PATCH] scripts: enhance run.py to allow running OSv on arm

[Waldek Kozaczuk]

As you pointed out there are still many issues with supporting ARM by 
run.py. I will be improving it as we go.

On Sunday, February 23, 2020 at 3:31:19 AM UTC-5, Nadav Har'El wrote:
>
> I'll commit, but have some questions below, for future improvements.
>
> --
> Nadav Har'El
> n...@scylladb.com 
>
>
> On Sat, Feb 22, 2020 at 11:13 PM Waldemar Kozaczuk  > wrote:
>
>> Signed-off-by: Waldemar Kozaczuk >
>> ---
>>  scripts/run.py | 53 +-
>>  1 file changed, 39 insertions(+), 14 deletions(-)
>>
>> diff --git a/scripts/run.py b/scripts/run.py
>> index f4452345..5a86c297 100755
>> --- a/scripts/run.py
>> +++ b/scripts/run.py
>> @@ -71,8 +71,11 @@ def set_imgargs(options):
>>  if options.hypervisor == 'qemu_microvm':
>>  execute = '--nopci ' + execute
>>
>> +if options.arch == 'aarch64':
>> +execute = '--nomount --maxnic=0 ' + execute #TODO: Investigate 
>> why both virtio-blk and virtio-net does not seem to work
>> +
>>  options.osv_cmdline = execute
>> -if options.kernel or options.hypervisor == 'qemu_microvm':
>> +if options.kernel or options.hypervisor == 'qemu_microvm' or 
>> options.arch == 'aarch64':
>>  return
>>
>>  cmdline = [os.path.join(osv_base, "scripts/imgedit.py"), "setargs", 
>> options.image_file, execute]
>> @@ -103,11 +106,16 @@ def start_osv_qemu(options):
>>  else:
>>  aio = 'cache=none,aio=native'
>>
>> -args = [
>> -"-m", options.memsize,
>> -"-smp", options.vcpus]
>> +if options.arch == 'aarch64':
>> +args = [
>> +"-m", "1G",
>> +"-smp", "2"]
>>
>
> This is strange, why for aarch64 you cannot obey the regular memsize and 
> vcpus option and silently ignore them?
>
> +else:
>> +args = [
>> +"-m", options.memsize,
>> +"-smp", options.vcpus]
>>
>> -if not options.novnc and options.hypervisor != 'qemu_microvm':
>> +if not options.novnc and options.hypervisor != 'qemu_microvm' and 
>> options.arch == 'x86_64':
>>  args += [
>>  "-vnc", options.vnc]
>>  else:
>> @@ -122,7 +130,7 @@ def start_osv_qemu(options):
>>  args += [
>>  "-display", "sdl"]
>>
>> -if options.kernel or options.hypervisor == 'qemu_microvm':
>> +if options.kernel or options.hypervisor == 'qemu_microvm' or 
>> options.arch == 'aarch64':
>>  boot_index = ""
>>  args += [
>>  "-kernel", options.kernel_file,
>> @@ -130,7 +138,12 @@ def start_osv_qemu(options):
>>  else:
>>  boot_index = ",bootindex=0"
>>
>> -if options.hypervisor == 'qemu_microvm':
>> +if options.arch == 'aarch64':
>> +args += [
>> +"-machine", "virt", "-machine", "gic-version=2", "-cpu", 
>> "cortex-a57",
>> +"-device", "virtio-blk-device,id=blk0,drive=hd0,scsi=off%s%s" % 
>> (boot_index, options.virtio_device_suffix),
>> +"-drive", "file=%s,if=none,id=hd0,%s" % (options.image_file, 
>> aio)]
>> +elif options.hypervisor == 'qemu_microvm':
>>  args += [
>>  "-M", "microvm,x-option-roms=off,pit=off,pic=off,rtc=off",
>>  "-nodefaults", "-no-user-config", "-no-reboot", "-global", 
>> "virtio-mmio.force-legacy=off",
>> @@ -219,11 +232,13 @@ def start_osv_qemu(options):
>>  args += ["-serial", "stdio"]
>>  elif options.detach:
>>  args += ["-daemonize"]
>> -else:
>> +elif options.arch == 'x86_64':
>>  signal_option = ('off', 'on')[options.with_signals]
>>  args += ["-chardev", "stdio,mux=on,id=stdio,signal=%s" % 
>> signal_option]
>>  args += ["-mon", "chardev=stdio,mode=readline"]
>>  args += ["-device", "isa-serial,chardev=stdio"]
>> +else:
>> +pass
>>
>>  for a in options.pass_args or []:
>>  args += a.split()
>> @@ -236,7 +251,11 @@ def start_osv_qemu(options):
>>  qemu_env = os.environ.copy()
>>
>>  qemu_env['OSV_BRIDGE'] = options.bridge
>> -cmdline = [options.qemu_path] + args
>> +if options.arch == 'aarch64':
>> +qemu_path = 'qemu-system-aarch64'
>> +else:
>> +qemu_path = options.qemu_path
>> +cmdline = [qemu_path] + args
>>  if options.dry_run:
>>  print(format_args(cmdline))
>>  else:
>> @@ -245,7 +264,7 @@ def start_osv_qemu(options):
>>  sys.exit("qemu failed.")
>>  except OSError as e:
>>  if e.errno == errno.ENOENT:
>> -print("'qemu-system-x86_64' binary not found. Please install 
>> the qemu-system-x86 package.")
>> +print("'%s' binary not found. Please install the 
>> qemu-system-x86 package." % qemu_path)
>>
>
> The name of the package you are suggesting to install should also differ 
> for the different binary...
>
>  else:
>>  print("OS error({0}): \"{1}\" while running 
>> qemu-system-x86_64 {2}".
>>  format(e.errno, e.strerror, 

Re: [osv-dev] [PATCH V2] scripts: upgrade to python 3 - part 3

[Waldek Kozaczuk]

On Sunday, February 23, 2020 at 3:20:10 AM UTC-5, Nadav Har'El wrote:
>
> On Sat, Feb 22, 2020 at 4:20 PM Waldemar Kozaczuk  > wrote:
>
>> This is the third and the last batch of the python 3 upgrade changes and
>> focuses on debug (loader.py) and trace related scripts.
>>
>> Following scripts have either been adapted to version 3 or
>> verified that they do NOT need to be changed by all patches the series:
>>
> ./scripts/export_manifest.py
>>
>
> Please note that most or not all of the scripts that have 
> "#!/usr/bin/python" on top
> (not python2 or python3) have been tested for years by different people 
> who had either
> python2 or python3 as their default "python", and should ideally be 
> working for both.
> That's especially true in the scripts used in regular scripts/build, which 
> people would 
> have noticed if they were broken.
> So it's indeed worth testing that this is in fact true, but the last thing 
> I want to see is
> rushing patches to "convert" a script that already works on Python3 to 
> Python3.
>
>
> Signed-off-by: Waldemar Kozaczuk >
>> ---
>>  scripts/loader.py|  5 +--
>>  scripts/osv/debug.py |  3 +-
>>  scripts/osv/prof.py  | 16 ++---
>>  scripts/osv/trace.py | 45 ++--
>>  scripts/osv/tree.py  |  4 +--
>>  scripts/trace.py | 84 
>>  6 files changed, 90 insertions(+), 67 deletions(-)
>>
>> diff --git a/scripts/loader.py b/scripts/loader.py
>> index 500d864a..4b82fd4a 100644
>> --- a/scripts/loader.py
>> +++ b/scripts/loader.py
>> @@ -1,4 +1,4 @@
>> -#!/usr/bin/python2
>> +#!/usr/bin/python
>>
>>  import gdb
>>  import re
>> @@ -1034,7 +1034,7 @@ class osv_info_callouts(gdb.Command):
>>  fname = callout['c_fn']
>>
>>  # time
>> -t = int(callout['c_to_ns'])
>> +t = int(callout['c_to_ns']['__d']['__r'])
>>
>
> Thanks, this was an already-needed fix (not related to Python 3). It was 
> probably casued by changing C++
> ABI, and while this change is good for us, it may not work for people 
> using older version of the compiler.
> Traditionally we handled this with code like:
>
> try:
> return self.map_header['_M_bbegin']
> except gdb.error:
> return self.map_header['_M_before_begin']['_M_nxt']
>
> Would be good to do this here too.
> But I'll not block this commit. I'll commit it as-is, and for your 
> consideration if you want to submit a followup patch to make this change 
> backward-compatible with older compilers.
>
I am about to send a patch accommodating your request. 

I think we have a similar issue with this code (probably line in bold):

def runqueue(cpuid, node=None):

if node == None:

cpus = gdb.lookup_global_symbol('sched::cpus').value()

cpu = cpus['_M_impl']['_M_start'][cpuid]

rq = cpu['runqueue']

*p = rq['data_']['node_plus_pred_']*

node = p['header_plus_size_']['header_']['parent_']


if node:

offset = gdb.parse_and_eval('(int)&((sched::thread 
*)0)->_runqueue_link')

thread = node.cast(gdb.lookup_type('void').pointer()) - offset

thread = thread.cast(gdb.lookup_type('sched::thread').pointer())


for x in runqueue(cpuid, node['left_']):

yield x


yield thread


for x in runqueue(cpuid, node['right_']):

yield x
 
Trying to run 'osv runqueue' gives this error:

 osv runqueue

CPU 0:

Python Exception  There is no member or method named 
data_.: 

Error occurred in Python: There is no member or method named data_.

I could not quite figure out how exactly the 'runqueue' has changed - it 
does not seem to have '_data' field anymore. Do you have any idea how we 
should fix it?

>
>
>>  # flags
>>  CALLOUT_ACTIVE = 0x0002
>> @@ -1176,6 +1176,7 @@ def all_traces():
>>  max_trace = ulong(trace_buffer['_size'])
>>
>>  if not trace_log_base:
>> +print('!!! Could not find any trace data! Make sure 
>> "--trace" option matches some tracepoints.')
>>  raise StopIteration
>>
>>  trace_log = inf.read_memory(trace_log_base, max_trace)
>> diff --git a/scripts/osv/debug.py b/scripts/osv/debug.py
>> index fe42be60..83372ada 100644
>> --- a/scripts/osv/debug.py
>> +++ b/scripts/osv/debug.py
>> @@ -38,7 +38,7 @@ class SymbolResolver(object):
>>  if show_inline:
>>  flags += 'i'
>>  self.addr2line = subprocess.Popen(['addr2line', '-e', 
>> object_path, flags],
>> -stdin=subprocess.PIPE, stdout=subprocess.PIPE)
>> +stdin=subprocess.PIPE, stdout=subprocess.PIPE, text=True)
>>  self.cache = {}
>>
>>  def next_line(self):
>> @@ -82,6 +82,7 @@ class SymbolResolver(object):
>>  if self.show_inline:
>>  self.addr2line.stdin.write('0\n')
>>
>> +self.addr2line.stdin.flush()
>>  result = self.parse_line(addr, self.next_line())
>>
>>  

[osv-dev] Jenkins build is back to normal : osv-build #1570

[jenkins]

See 


-- 
You received this message because you are subscribed to the Google Groups "OSv 
Development" group.
To unsubscribe from this group and stop receiving emails from it, send an email 
to osv-dev+unsubscr...@googlegroups.com.
To view this discussion on the web visit 
https://groups.google.com/d/msgid/osv-dev/926533248.1070.1582553801116.JavaMail.jenkins%40jenkins.cloudius-systems.com.

[osv-dev] Jenkins build is back to normal : osv-build-nightly #2072

[jenkins]

See 


-- 
You received this message because you are subscribed to the Google Groups "OSv 
Development" group.
To unsubscribe from this group and stop receiving emails from it, send an email 
to osv-dev+unsubscr...@googlegroups.com.
To view this discussion on the web visit 
https://groups.google.com/d/msgid/osv-dev/132159863.1057.1582550651294.JavaMail.jenkins%40jenkins.cloudius-systems.com.

[osv-dev] [COMMIT osv master] scripts/setup.py: fix Fedora package to python3

[Commit Bot]

From: Nadav Har'El 
Committer: Nadav Har'El 
Branch: master

scripts/setup.py: fix Fedora package to python3

In Fedora, the "python-*" packages may refer to Python2, so we need to
install the "python3-*" packages specifically, now that our scripts
require Python3.

This patch fixes build failures on Fedora 29 and Fedora 30.

Debian/Ubuntu setups may need similar fixes, but I didn't test them and
a fix is not included in this patch.

Signed-off-by: Nadav Har'El 

---
diff --git a/scripts/setup.py b/scripts/setup.py
--- a/scripts/setup.py
+++ b/scripts/setup.py
@@ -51,7 +51,7 @@ class Fedora(object):
 'openssl-libs',
 'p11-kit',
 'patch',
-'python-dpkt',
+'python3-dpkt',
 'qemu-img',
 'qemu-system-x86',
 'tcpdump',

-- 
You received this message because you are subscribed to the Google Groups "OSv 
Development" group.
To unsubscribe from this group and stop receiving emails from it, send an email 
to osv-dev+unsubscr...@googlegroups.com.
To view this discussion on the web visit 
https://groups.google.com/d/msgid/osv-dev/6d6df3059f515afa%40google.com.

Re: [osv-dev] Build failed in Jenkins: osv-build #1569

[Nadav Har'El]

The fix to this is simple - setup.py needs to install the python3-dpkt
package, not python-dpkt.
Will push a patch now.

--
Nadav Har'El
n...@scylladb.com


On Mon, Feb 24, 2020 at 4:48 AM  wrote:

> See <
> https://jenkins.scylladb.com/job/osv-build/1569/display/redirect?page=changes
> >
>
> Changes:
>
> [jwkozaczuk] scripts: upgrade to python 3 - part 3
>
> [jwkozaczuk] scripts: enhance run.py to allow running OSv on arm
>
>
> --
> [...truncated 93.01 KB...]
> Adding /tests/ClassPutInRoot.class...
> Adding /usr/lib/jvm/java/jre/lib/ext/tests-jre-extension.jar...
> Adding /usr/lib/libtest_simple.so...
> Adding /usr/lib/libtest_empty.so...
> Adding /usr/lib/libtest_dlsym_from_this_grandchild.so...
> Adding /usr/lib/libtest_dlsym_from_this_child.so...
> Adding /usr/lib/libtest_dlsym_from_this.so...
> Adding /usr/lib/libdlext_test.so...
> Adding /usr/lib/libtest_with_dependency.so...
> Adding /usr/lib/libtest_check_rtld_next_from_library.so...
> Adding /tests/libtest_simple.so...
> Adding /tests/libtest_empty.so...
> Adding /tests/libtest_dlsym_from_this_grandchild.so...
> Adding /tests/libtest_dlsym_from_this_child.so...
> Adding /tests/libtest_dlsym_from_this.so...
> Adding /tests/libdlext_test.so...
> Adding /tests/libtest_with_dependency.so...
> Adding /tests/libtest_check_rtld_next_from_library.so...
> Adding /testrunner.so...
> Adding /usr/lib/libboost_chrono.so.1.69.0...
> Adding /usr/lib/libboost_filesystem.so.1.69.0...
> Adding /usr/lib/libboost_system.so.1.69.0...
> Adding /usr/lib/libboost_timer.so.1.69.0...
> Adding /usr/lib/libboost_unit_test_framework.so.1.69.0...
> Adding /tests/tst-pthread.so...
> Adding /tests/misc-ramdisk.so...
> Adding /tests/tst-vblk.so...
> Adding /tests/tst-bsd-evh.so...
> Adding /tests/misc-bsd-callout.so...
> Adding /tests/tst-bsd-kthread.so...
> Adding /tests/tst-bsd-taskqueue.so...
> Adding /tests/tst-fpu.so...
> Adding /tests/tst-preempt.so...
> Adding /tests/tst-tracepoint.so...
> Adding /tests/tst-hub.so...
> Adding /tests/misc-console.so...
> Adding /tests/misc-leak.so...
> Adding /tests/misc-readbench.so...
> Adding /tests/misc-mmap-anon-perf.so...
> Adding /tests/tst-mmap-file.so...
> Adding /tests/misc-mmap-big-file.so...
> Adding /tests/tst-mmap.so...
> Adding /tests/tst-huge.so...
> Adding /tests/tst-elf-permissions.so...
> Adding /tests/misc-mutex.so...
> Adding /tests/misc-sockets.so...
> Adding /tests/tst-condvar.so...
> Adding /tests/tst-queue-mpsc.so...
> Adding /tests/tst-af-local.so...
> Adding /tests/tst-pipe.so...
> Adding /tests/tst-yield.so...
> Adding /tests/misc-ctxsw.so...
> Adding /tests/tst-read.so...
> Adding /tests/tst-symlink.so...
> Adding /tests/tst-openat.so...
> Adding /tests/tst-eventfd.so...
> Adding /tests/tst-remove.so...
> Adding /tests/misc-wake.so...
> Adding /tests/tst-epoll.so...
> Adding /tests/misc-lfring.so...
> Adding /tests/misc-fsx.so...
> Adding /tests/tst-sleep.so...
> Adding /tests/tst-resolve.so...
> Adding /tests/tst-except.so...
> Adding /tests/misc-tcp-sendonly.so...
> Adding /tests/tst-tcp-nbwrite.so...
> Adding /tests/misc-tcp-hash-srv.so...
> Adding /tests/misc-loadbalance.so...
> Adding /tests/misc-scheduler.so...
> Adding /tests/tst-console.so...
> Adding /tests/tst-app.so...
> Adding /tests/misc-setpriority.so...
> Adding /tests/misc-timeslice.so...
> Adding /tests/misc-tls.so...
> Adding /tests/misc-gtod.so...
> Adding /tests/tst-dns-resolver.so...
> Adding /tests/tst-kill.so...
> Adding /tests/tst-truncate.so...
> Adding /tests/misc-panic.so...
> Adding /tests/tst-utimes.so...
> Adding /tests/tst-utimensat.so...
> Adding /tests/tst-futimesat.so...
> Adding /tests/misc-tcp.so...
> Adding /tests/tst-strerror_r.so...
> Adding /tests/misc-random.so...
> Adding /tests/misc-urandom.so...
> Adding /tests/tst-commands.so...
> Adding /tests/tst-options.so...
> Adding /tests/tst-threadcomplete.so...
> Adding /tests/tst-timerfd.so...
> Adding /tests/tst-nway-merger.so...
> Adding /tests/tst-memmove.so...
> Adding /tests/tst-pthread-clock.so...
> Adding /tests/misc-procfs.so...
> Adding /tests/tst-chdir.so...
> Adding /tests/tst-chmod.so...
> Adding /tests/tst-hello.so...
> Adding /tests/misc-concurrent-io.so...
> Adding /tests/tst-concurrent-init.so...
> Adding /tests/tst-ring-spsc-wraparound.so...
> Adding /tests/tst-shm.so...
> Adding /tests/tst-align.so...
> Adding /tests/tst-cxxlocale.so...
> Adding /tests/misc-tcp-close-without-reading.so...
> Adding /tests/tst-sigwait.so...
> Adding /tests/tst-sampler.so...
> Adding /tests/misc-malloc.so...
> Adding /tests/misc-memcpy.so...
> Adding /tests/misc-free-perf.so...
> Adding /tests/misc-printf.so...
> Adding /tests/tst-hostname.so...
> Adding /tests/tst-sendfile.so...
> Adding /tests/misc-lock-perf.so...
> Adding /tests/tst-uio.so...
> Adding /tests/tst-printf.so...
> Adding /tests/tst-pthread-affinity.so...
> Adding /tests/tst-pthread-tsd.so...
> Adding /tests/tst-thread-local.so...
> Adding /tests/tst-zfs-mount.so...
>