[osv-dev] [PATCH] java: eliminate openjdk7

[Waldemar Kozaczuk]

Java 7 reached end of life almost 5 years ago and has since
been largely replaced by Java 8 which is still widely used.

This patch eliminates openjdk7 module and makes openjdk8-zull-full
a default java module. This patch also replaces dependencies in the
main makefile on antique openjdk7 in external/x64 with openjdk8
from host that should be installed by setup.py.

References #743

Signed-off-by: Waldemar Kozaczuk 
---
 Makefile   | 14 +++---
 modules/httpserver-jvm-plugin/Makefile |  2 --
 modules/java-base/common.gmk   |  1 +
 modules/java/module.py |  2 +-
 modules/openjdk7/module.py | 22 --
 scripts/build  | 11 +--
 6 files changed, 14 insertions(+), 38 deletions(-)
 delete mode 100644 modules/openjdk7/module.py

diff --git a/Makefile b/Makefile
index 62277d5f..8886d1ec 100644
--- a/Makefile
+++ b/Makefile
@@ -112,7 +112,6 @@ endif
 #   musl/ -  for some of the header files (symbolic links in include/api) and
 #some of the source files ($(musl) below).
 #   external/x64/acpica - for the ACPICA library (see $(acpi) below).
-#   external/x64/openjdk.bin - for $(java-targets) below.
 # Additional submodules are need when certain make parameters are used.
 ifeq (,$(wildcard musl/include))
 $(error Missing musl/ directory. Please run "git submodule update --init 
--recursive")
@@ -120,9 +119,6 @@ endif
 ifeq (,$(wildcard external/x64/acpica/source))
 $(error Missing external/x64/acpica/ directory. Please run "git submodule 
update --init --recursive")
 endif
-ifeq (,$(wildcard external/x64/openjdk.bin/usr))
-$(error Missing external/x64/openjdk.bin/ directory. Please run "git 
submodule update --init --recursive")
-endif
 
 # This makefile wraps all commands with the $(quiet) or $(very-quiet) macros
 # so that instead of half-a-screen-long command lines we short summaries
@@ -233,9 +229,13 @@ INCLUDES += -isystem include/glibc-compat
 
 gccbase = external/$(arch)/gcc.bin
 miscbase = external/$(arch)/misc.bin
-jdkbase := $(shell find external/$(arch)/openjdk.bin/usr/lib/jvm \
- -maxdepth 1 -type d -name 'java*')
 
+ifeq ($(arch),x64)
+  jdkbase := $(dir $(shell readlink -f $$(which javac)))/..
+else
+  jdkbase := $(shell find external/$(arch)/openjdk.bin/usr/lib/jvm \
+   -maxdepth 1 -type d -name 'java*')
+endif
 
 ifeq ($(gcc_include_env), external)
   gcc-inc-base := $(dir $(shell find $(gccbase)/ -name vector | grep -v -e 
debug/vector$$ -e profile/vector$$))
@@ -1940,7 +1940,7 @@ $(bootfs_manifest_dep): phony
 $(out)/bootfs.bin: scripts/mkbootfs.py $(bootfs_manifest) 
$(bootfs_manifest_dep) $(tools:%=$(out)/%) \
$(out)/zpool.so $(out)/zfs.so $(out)/libenviron.so 
$(out)/libvdso.so
$(call quiet, olddir=`pwd`; cd $(out); "$$olddir"/scripts/mkbootfs.py 
-o bootfs.bin -d bootfs.bin.d -m "$$olddir"/$(bootfs_manifest) \
-   -D jdkbase="$$olddir"/$(jdkbase) -D 
gccbase="$$olddir"/$(gccbase) \
+   -D gccbase="$$olddir"/$(gccbase) \
-D miscbase="$$olddir"/$(miscbase), MKBOOTFS $@)
 
 $(out)/bootfs.o: $(out)/bootfs.bin
diff --git a/modules/httpserver-jvm-plugin/Makefile 
b/modules/httpserver-jvm-plugin/Makefile
index fdc20bb7..bb802720 100644
--- a/modules/httpserver-jvm-plugin/Makefile
+++ b/modules/httpserver-jvm-plugin/Makefile
@@ -1,7 +1,5 @@
-
 INCLUDES = -I$(src)/build/$(mode)/gen/include
 INCLUDES += -I../../include -I. -I../../java -I../../arch/$(ARCH) -I../..
-INCLUDES += -I$(jdkbase)/include -I$(jdkbase)/include/linux
 INCLUDES += -I../httpserver-api
 
 # compiler flags:
diff --git a/modules/java-base/common.gmk b/modules/java-base/common.gmk
index e54a301a..1c79a6b5 100644
--- a/modules/java-base/common.gmk
+++ b/modules/java-base/common.gmk
@@ -1,3 +1,4 @@
+jdkbase = $(dir $(shell readlink -f $$(which javac)))/..
 
 INCLUDES = -I$(src)/arch/$(arch) -I$(src) -I$(src)/include -I$(src)/arch/common
 INCLUDES += -I$(src)/include/glibc-compat
diff --git a/modules/java/module.py b/modules/java/module.py
index 7af25104..0c0eb6c5 100644
--- a/modules/java/module.py
+++ b/modules/java/module.py
@@ -1,4 +1,4 @@
 from osv.modules import api
 
 api.require('java-non-isolated')
-api.require('openjdk7')
+api.require('openjdk8-zulu-full')
diff --git a/modules/openjdk7/module.py b/modules/openjdk7/module.py
deleted file mode 100644
index e4895465..
--- a/modules/openjdk7/module.py
+++ /dev/null
@@ -1,22 +0,0 @@
-from osv.modules.filemap import FileMap
-from osv.modules import api
-import os, os.path
-
-api.require('java-cmd')
-provides = ['java']
-
-usr_files = FileMap()
-
-jdkdir = os.path.basename(os.path.expandvars('${jdkbase}'))
-
-usr_files.add('${jdkbase}').to('/usr/lib/jvm/java') \
-.include('lib/**') \
-.include('jre/**') \
-.include('bin/java') \
-.exclude('jre/lib/security/cacerts') \
-.exclude('jre/lib/audio/**')
-

[osv-dev] [COMMIT osv master] externals: eliminate unused glibcbase build parameter

[Commit Bot]

From: Waldemar Kozaczuk 
Committer: Waldemar Kozaczuk 
Branch: master

externals: eliminate unused glibcbase build parameter

References #743

Signed-off-by: Waldemar Kozaczuk 

---
diff --git a/Makefile b/Makefile
--- a/Makefile
+++ b/Makefile
@@ -231,7 +231,6 @@ local-includes =
 INCLUDES = $(local-includes) -Iarch/$(arch) -I. -Iinclude  -Iarch/common
 INCLUDES += -isystem include/glibc-compat

-glibcbase = external/$(arch)/glibc.bin
 gccbase = external/$(arch)/gcc.bin
 miscbase = external/$(arch)/misc.bin
 jdkbase := $(shell find external/$(arch)/openjdk.bin/usr/lib/jvm \
@@ -1941,8 +1940,8 @@ $(bootfs_manifest_dep): phony
 $(out)/bootfs.bin: scripts/mkbootfs.py $(bootfs_manifest)  
$(bootfs_manifest_dep) $(tools:%=$(out)/%) \

$(out)/zpool.so $(out)/zfs.so $(out)/libenviron.so 
$(out)/libvdso.so
 	$(call quiet, olddir=`pwd`; cd $(out); "$$olddir"/scripts/mkbootfs.py -o  
bootfs.bin -d bootfs.bin.d -m "$$olddir"/$(bootfs_manifest) \

-   -D jdkbase="$$olddir"/$(jdkbase) -D 
gccbase="$$olddir"/$(gccbase) -D \
-		glibcbase="$$olddir"/$(glibcbase) -D miscbase="$$olddir"/$(miscbase),  
MKBOOTFS $@)

+   -D jdkbase="$$olddir"/$(jdkbase) -D 
gccbase="$$olddir"/$(gccbase) \
+   -D miscbase="$$olddir"/$(miscbase), MKBOOTFS $@)

 $(out)/bootfs.o: $(out)/bootfs.bin
 $(out)/bootfs.o: ASFLAGS += -I$(out)
diff --git a/scripts/build b/scripts/build
--- a/scripts/build
+++ b/scripts/build
@@ -172,7 +172,6 @@ modules=${vars[modules]-!$image}
 # TODO: some modules need these... Would be better if they wouldn't...
  
jdkbase=${vars[jdkbase]-`find "$SRC"/external/$arch/openjdk.bin/usr/lib/jvm  
-maxdepth 1 -type d -name 'java*'`}

 gccbase=${vars[gccbase]-"$SRC"/external/$arch/gcc.bin}
-glibcbase=${vars[glibcbase]-"$SRC"/external/$arch/glibc.bin}
 miscbase=${vars[miscbase]-"$SRC"/external/$arch/misc.bin}

 case $OUT in
@@ -270,15 +269,15 @@ zfs)

if [ "$export" == "none" ]
then
-		"$SRC"/scripts/upload_manifest.py -o usr.img -m usr.manifest -D  
jdkbase="$jdkbase" -D gccbase="$gccbase" -D glibcbase="$glibcbase" -D  
miscbase="$miscbase"
+		"$SRC"/scripts/upload_manifest.py -o usr.img -m usr.manifest -D  
jdkbase="$jdkbase" -D gccbase="$gccbase" -D miscbase="$miscbase"

else
export_dir=${vars[export_dir]-$SRC/build/export}
-		"$SRC"/scripts/export_manifest.py -e "$export_dir" -m usr.manifest -D  
jdkbase="$jdkbase" -D gccbase="$gccbase" -D glibcbase="$glibcbase" -D  
miscbase="$miscbase"
+		"$SRC"/scripts/export_manifest.py -e "$export_dir" -m usr.manifest -D  
jdkbase="$jdkbase" -D gccbase="$gccbase" -D miscbase="$miscbase"

fi
;;
 rofs)
rm -rf rofs.img
-	"$SRC"/scripts/gen-rofs-img.py -o rofs.img -m usr.manifest -D  
jdkbase="$jdkbase" -D gccbase="$gccbase" -D glibcbase="$glibcbase" -D  
miscbase="$miscbase"
+	"$SRC"/scripts/gen-rofs-img.py -o rofs.img -m usr.manifest -D  
jdkbase="$jdkbase" -D gccbase="$gccbase" -D miscbase="$miscbase"

rofs_size=`stat --printf %s rofs.img`
img_size=$((kernel_end + rofs_size))
cp loader.img bare.raw

--
You received this message because you are subscribed to the Google Groups "OSv 
Development" group.
To unsubscribe from this group and stop receiving emails from it, send an email 
to osv-dev+unsubscr...@googlegroups.com.
To view this discussion on the web visit 
https://groups.google.com/d/msgid/osv-dev/156244059c251f04%40google.com.

[osv-dev] [COMMIT osv master] httpserver: use libyaml header from host instead of external

[Commit Bot]

From: Waldemar Kozaczuk 
Committer: Waldemar Kozaczuk 
Branch: master

httpserver: use libyaml header from host instead of external

Signed-off-by: Waldemar Kozaczuk 

---
diff --git a/modules/httpserver-api/global_server.hh  
b/modules/httpserver-api/global_server.hh

--- a/modules/httpserver-api/global_server.hh
+++ b/modules/httpserver-api/global_server.hh
@@ -12,7 +12,7 @@
 #include 
 #include 
 #include 
-#include 
+#include 

 namespace httpserver {
 /**

--
You received this message because you are subscribed to the Google Groups "OSv 
Development" group.
To unsubscribe from this group and stop receiving emails from it, send an email 
to osv-dev+unsubscr...@googlegroups.com.
To view this discussion on the web visit 
https://groups.google.com/d/msgid/osv-dev/001921059c251f29%40google.com.

[osv-dev] [COMMIT osv master] elf: separate loading segments from processing headers

[Commit Bot]

From: Waldemar Kozaczuk 
Committer: Nadav Har'El 
Branch: master

elf: separate loading segments from processing headers

The issue #1045 exposed a bug in the dynamic linker when processing
some headers like PT_NOTE require accessing data in the PT_LOAD segments.
If PT_NOTE headers come earlier than PT_NOTE we end up with the page fault
as the portions of ELF file have not been mapped yet.

This patch separates loading of segments - load_segments() - from
processing headers by adding new method - process_headers().

Fixes #1045

Signed-off-by: Waldemar Kozaczuk 
Message-Id: <20200114125901.16800-1-jwkozac...@gmail.com>

---
diff --git a/core/elf.cc b/core/elf.cc
--- a/core/elf.cc
+++ b/core/elf.cc
@@ -452,14 +452,22 @@ extern "C" char _pie_static_tls_start,  
_pie_static_tls_end;

 void object::load_segments()
 {
 elf_debug("Loading segments\n");
+for (unsigned i = 0; i < _ehdr.e_phnum; ++i) {
+auto  = _phdrs[i];
+if (phdr.p_type == PT_LOAD) {
+load_segment(phdr);
+}
+}
+}
+
+void object::process_headers()
+{
+elf_debug("Processing headers\n");
 for (unsigned i = 0; i < _ehdr.e_phnum; ++i) {
 auto  = _phdrs[i];
 switch (phdr.p_type) {
 case PT_NULL:
 break;
-case PT_LOAD:
-load_segment(phdr);
-break;
 case PT_DYNAMIC:
 _dynamic_table = reinterpret_cast(_base +  
phdr.p_vaddr);

 break;
@@ -495,6 +503,7 @@ void object::load_segments()
 }
 break;
 }
+case PT_LOAD:
 case PT_PHDR:
 case PT_GNU_STACK:
 case PT_GNU_RELRO:
@@ -1233,6 +1242,7 @@ program::program(void* addr)
 _core->set_base(program_base);
 assert(_core->module_index() == core_module_index);
 _core->load_segments();
+_core->process_headers();
 set_search_path({"/", "/usr/lib"});
 // Our kernel already supplies the features of a bunch of traditional
 // shared libraries:
@@ -1361,6 +1371,7 @@ program::load_object(std::string name,  
std::vector extra_path,

 _modules_rcu.assign(new_modules.release());
 osv::rcu_dispose(old_modules);
 ef->load_segments();
+ef->process_headers();
 if (!ef->is_non_pie_executable())
_next_alloc = ef->end();
 add_debugger_obj(ef.get());
diff --git a/include/osv/elf.hh b/include/osv/elf.hh
--- a/include/osv/elf.hh
+++ b/include/osv/elf.hh
@@ -348,6 +348,7 @@ public:
 void* end() const;
 Elf64_Sym* lookup_symbol(const char* name, bool self_lookup);
 void load_segments();
+void process_headers();
 void unload_segments();
 void fix_permissions();
 void* resolve_pltgot(unsigned index);

--
You received this message because you are subscribed to the Google Groups "OSv 
Development" group.
To unsubscribe from this group and stop receiving emails from it, send an email 
to osv-dev+unsubscr...@googlegroups.com.
To view this discussion on the web visit 
https://groups.google.com/d/msgid/osv-dev/40843b059c1df478%40google.com.

Re: [osv-dev] Fail to read data from large file

[Dor Laor]

A classical good OSS reporter and a good (!) committer

On Tue, Jan 14, 2020 at 2:47 AM Nadav Har'El  wrote:
>
> On Tue, Jan 14, 2020 at 12:41 PM Lewis Tian  wrote:
>>
>>
>>
>> On Tuesday, January 14, 2020 at 4:45:27 PM UTC+8, Nadav Har'El wrote:
>>>
>>>
>>> Interesting. Smells like a serious stdio bug that needs to be debugged :-(
>>> I think it's not a coincidence that position is 1025, with stdio's 
>>> BUFSIZ=1024.
>>>
>>> Just as a completely wild guess, can you please try if the following patch 
>>> to libc/internal/shgetc.c helps?
>>>
>>> @@ -22,5 +22,6 @@
>>>   else
>>>   f->shend = f->rend;
>>>   if (f->rend) f->shcnt += f->rend - f->rpos + 1;
>>> + if (f->rpos[-1] != c) f->rpos[-1] = c;
>>>   return c;
>>>  }
>>>
>>
>> Yeah, the patch is helpful! Thanks! : )
>
>
> I'm pleasantly surprised :-) I'll send a more organized patch to the mailing 
> list.
>>
>>



>
>
>>
>> When I use a small graph (4 nodes, 7 edges), it runs normally.
>>
>> taseikyo@ubuntu:~/Desktop/osv$ ./scripts/build image=my-pagerank-test
>> taseikyo@ubuntu:~/Desktop/osv$ ./scripts/run.py
>> OSv v0.54.0-71-g69a0ce39
>> eth0: 192.168.122.15
>> Booted up in 356.44 ms
>> Cmdline: /pagerank
>>
>> Graph data:
>>
>>   Nodes: 4, Edges: 7
>>
>>
>> Number of iteration to converge: 41
>>
>> Final Pagerank values:
>>
>> [0.159913 , 0.144016 , 0.144016 , 0.082809 ]
>>
>> Time spent: 0.693802 seconds.
>>
>> Is osv unable to read large files (bug?) I'll appreciate your help very 
>> much! : )
>>
>> --
>> You received this message because you are subscribed to the Google 
>> Groups "OSv Development" group.
>> To unsubscribe from this group and stop receiving emails from it, send 
>> an email to osv...@googlegroups.com.
>> To view this discussion on the web visit 
>> https://groups.google.com/d/msgid/osv-dev/50f3a073-e364-484c-803e-570e0dd6530c%40googlegroups.com.

 --
 You received this message because you are subscribed to the Google Groups 
 "OSv Development" group.
 To unsubscribe from this group and stop receiving emails from it, send an 
 email to osv...@googlegroups.com.
 To view this discussion on the web visit 
 https://groups.google.com/d/msgid/osv-dev/ae3d8e59-bb13-40a4-bdd0-b03ec89b9ff3%40googlegroups.com.
>>
>> --
>> You received this message because you are subscribed to the Google Groups 
>> "OSv Development" group.
>> To unsubscribe from this group and stop receiving emails from it, send an 
>> email to osv-dev+unsubscr...@googlegroups.com.
>> To view this discussion on the web visit 
>> https://groups.google.com/d/msgid/osv-dev/9e83516e-d25f-4409-a15d-a0a59871978d%40googlegroups.com.
>
> --
> You received this message because you are subscribed to the Google Groups 
> "OSv Development" group.
> To unsubscribe from this group and stop receiving emails from it, send an 
> email to osv-dev+unsubscr...@googlegroups.com.
> To view this discussion on the web visit 
> https://groups.google.com/d/msgid/osv-dev/CANEVyjva7W44mktoixKtdprxbEyXNt1qg7%3D%3D6qNatcH3xAGAfA%40mail.gmail.com.

-- 
You received this message because you are subscribed to the Google Groups "OSv 
Development" group.
To unsubscribe from this group and stop receiving emails from it, send an email 
to osv-dev+unsubscr...@googlegroups.com.
To view this discussion on the web visit 
https://groups.google.com/d/msgid/osv-dev/CAKUaUn7vw-e7stMfFxmUsJEuMqiQUp0KTAjxF6YuYDd0cW_edA%40mail.gmail.com.

[osv-dev] Re: [PATCH] Signed-off-by: BassMatt

[Waldek Kozaczuk]

Hi,

Would you mind re-submitting this patch with signature? Or giving us line:

Signed-off-by: Real Name  

so we can apply this patch?

Thanks in advance,
Waldek

On Thursday, December 12, 2019 at 3:02:54 PM UTC-5, BassMatt wrote:
>
> Main scripts in scripts/ folder updated to use Python3 
>
> I went through the scripts detailed in scripts/README and updated them to 
> use Python3. I used the Python "Future" module to provide suggestions, then 
> manually went through and applied the changes. The "Future" module gives 
> suggestions to allow for cross-compatibility between Python2/3, but since 
> it was expressed that only Python3 needed to be supported, I left all that 
> out. 
>
> The issue is detailed here: 
> https://github.com/cloudius-systems/osv/issues/1056 
> --- 
>  scripts/export_manifest.py | 14 +- 
>  scripts/firecracker.py |  4 +-- 
>  scripts/gen-rofs-img.py| 18 ++-- 
>  scripts/loader.py  | 57 ++ 
>  scripts/metadata.py|  6 ++-- 
>  scripts/mkbootfs.py|  7 ++--- 
>  scripts/module.py  |  4 +-- 
>  scripts/setup.py   | 10 +++ 
>  scripts/test.py|  6 ++-- 
>  scripts/tests/testing.py   |  4 +-- 
>  scripts/upload_manifest.py | 12 ++-- 
>  11 files changed, 65 insertions(+), 77 deletions(-) 
>
> diff --git a/scripts/export_manifest.py b/scripts/export_manifest.py 
> index 5e8fb4e7..370b1f29 100755 
> --- a/scripts/export_manifest.py 
> +++ b/scripts/export_manifest.py 
> @@ -1,4 +1,4 @@ 
> -#!/usr/bin/python 
> +#!/usr/bin/python3 
>   
>  import optparse, os, shutil 
>  from manifest_common import add_var, expand, unsymlink, read_manifest, 
> defines, strip_file 
> @@ -8,7 +8,7 @@ from manifest_common import add_var, expand, unsymlink, 
> read_manifest, defines, 
>  # support for links in OSv, e.g., /etc/mnttab: ->/proc/mounts. 
>  def export_package(manifest, dest): 
>  abs_dest = os.path.abspath(dest) 
> -print "[INFO] exporting into directory %s" % abs_dest 
> +print("[INFO] exporting into directory %s" % abs_dest) 
>   
>  # Remove and create the base directory where we are going to put all 
> package files. 
>  if os.path.exists(abs_dest): 
> @@ -39,7 +39,7 @@ def export_package(manifest, dest): 
>  os.makedirs(target_dir) 
>   
>  os.symlink(link_source, name) 
> -print "[INFO] added link %s -> %s" % (name, link_source) 
> +print("[INFO] added link %s -> %s" % (name, link_source)) 
>   
>  else: 
>  # If it is a symlink, then resolve it add to the list of host 
> symlinks to be created later 
> @@ -58,23 +58,23 @@ def export_package(manifest, dest): 
>  hostname = strip_file(hostname) 
>   
>  shutil.copy(hostname, name) 
> -print "[INFO] exported %s" % name 
> +print("[INFO] exported %s" % name) 
>  elif os.path.isdir(hostname): 
>  # If hostname is a dir, it is only a request to create 
> the folder on guest. Nothing to copy. 
>  if not os.path.exists(name): 
>  os.makedirs(name) 
> -print "[INFO] created dir %s" % name 
> +print("[INFO] created dir %s" % name) 
>   
>  else: 
>  # Inform the user that the rule cannot be applied. For 
> example, this happens for links in OSv. 
> -print "[ERR] unable to export %s" % hostname 
> +print("[ERR] unable to export %s" % hostname) 
>   
>  for link_source, name in host_symlinks: 
>  target_dir = os.path.dirname(name) 
>  if not os.path.exists(target_dir): 
>  os.makedirs(target_dir) 
>  os.symlink(link_source, name) 
> -print "[INFO] added link %s -> %s" % (name, link_source) 
> +print("[INFO] added link %s -> %s" % (name, link_source)) 
>   
>  def main(): 
>  make_option = optparse.make_option 
> diff --git a/scripts/firecracker.py b/scripts/firecracker.py 
> index 3f21081d..5c98b0ee 100755 
> --- a/scripts/firecracker.py 
> +++ b/scripts/firecracker.py 
> @@ -1,4 +1,4 @@ 
> -#!/usr/bin/env python 
> +#!/usr/bin/env python3 
>  # 
>  # pip install requests-unixsocket 
>  import sys 
> @@ -182,7 +182,7 @@ def find_firecracker(dirname): 
>  if not os.path.exists(firecracker_path): 
>  url_base = '
> https://github.com/firecracker-microvm/firecracker/releases/download' 
>  download_url = '%s/%s/firecracker-%s' % (url_base, 
> firecracker_version, firecracker_version) 
> -answer = raw_input("Firecracker executable has not been found 
> under %s. " 
> +answer = input("Firecracker executable has not been found under 
> %s. " 
> "Would you like to download it from %s and 
> place it under %s? [y|n]" % 
> (firecracker_path, download_url, 
> firecracker_path)) 
>  if 

Re: [osv-dev] [COMMIT osv master] procfs: add meminfo pseudo-file

[Waldek Kozaczuk]

On Sunday, January 12, 2020 at 5:43:48 AM UTC-5, Nadav Har'El wrote:
>
>
> On Fri, Jan 10, 2020 at 6:28 AM Commit Bot  > wrote:
>
>> From: Waldemar Kozaczuk >
>> Committer: Waldemar Kozaczuk >
>> Branch: master
>>
>> procfs: add meminfo pseudo-file
>>
>
> Can you please say a few words on the motivation of this change?
>
Needed for nodejs 12.  

> Apparently we already had /proc/meminfo, so what benefit does this change 
> bring?
>
We had sysfs meminfo version but not /proc/meminfo.

>
>
>> Signed-off-by: Waldemar Kozaczuk >
>>
>> ---
>> diff --git a/fs/procfs/procfs_vnops.cc b/fs/procfs/procfs_vnops.cc
>> --- a/fs/procfs/procfs_vnops.cc
>> +++ b/fs/procfs/procfs_vnops.cc
>> @@ -182,6 +182,7 @@ procfs_mount(mount* mp, const char *dev, int flags,  
>> const void* data)
>>   root->add("sys", sys);
>>
>>   root->add("cpuinfo", inode_count++, [] { return  
>> processor::features_str(); });
>> +root->add("meminfo", inode_count++, [] { return  
>> pseudofs::meminfo("MemTotal:\t%ld kB\nMemFree: \t%ld kB\n"); });
>>
>>   vp->v_data = static_cast(root);
>>
>> diff --git a/fs/pseudofs/pseudofs.cc b/fs/pseudofs/pseudofs.cc
>> --- a/fs/pseudofs/pseudofs.cc
>> +++ b/fs/pseudofs/pseudofs.cc
>> @@ -8,6 +8,7 @@
>>
>>   #include "pseudofs.hh"
>>   #include 
>> +#include 
>>
>>   namespace pseudofs {
>>
>> @@ -191,4 +192,14 @@ string cpumap()
>>   }
>>   return os.str();
>>   }
>> +
>> +string meminfo(const char* format)
>> +{
>> +struct sysinfo info;
>> +sysinfo();
>> +
>> +std::ostringstream os;
>> +osv::fprintf(os, format, info.totalram >> 10, info.freeram >> 10);
>> +return os.str();
>> +}
>>   }
>> diff --git a/fs/pseudofs/pseudofs.hh b/fs/pseudofs/pseudofs.hh
>> --- a/fs/pseudofs/pseudofs.hh
>> +++ b/fs/pseudofs/pseudofs.hh
>> @@ -146,6 +146,8 @@ int readdir(vnode *vp, file *fp, dirent *dir);
>>   int getattr(vnode *vp, vattr *attr);
>>
>>   string cpumap();
>> +
>> +string meminfo(const char* format);
>>   }
>>
>>   #endif
>> diff --git a/fs/sysfs/sysfs_vnops.cc b/fs/sysfs/sysfs_vnops.cc
>> --- a/fs/sysfs/sysfs_vnops.cc
>> +++ b/fs/sysfs/sysfs_vnops.cc
>> @@ -8,7 +8,6 @@
>>   #include 
>>   #include 
>>   #include 
>> -#include 
>>
>>   #include "fs/pseudofs/pseudofs.hh"
>>
>> @@ -21,18 +20,6 @@ static uint64_t inode_count = 1; /* inode 0 is 
>> reserved  
>> to root */
>>
>>   static mutex_t sysfs_mutex;
>>
>> -static string sysfs_meminfo()
>> -{
>> -struct sysinfo info;
>> -sysinfo();
>> -
>> -char total_line[200];
>> -sprintf(total_line, "Node 0 MemTotal:\t%ld kB\nNode 0 MemFree: 
>> \t%ld  
>> kB\n",
>> -info.totalram >> 10, info.freeram >> 10);
>> -
>> -return std::string(total_line);
>> -}
>> -
>>   static string sysfs_cpumap()
>>   {
>>   return pseudofs::cpumap() + "\n";
>> @@ -49,7 +36,7 @@ sysfs_mount(mount* mp, const char *dev, int flags, 
>> const  
>> void* data)
>>   auto* vp = mp->m_root->d_vnode;
>>
>>   auto node0 = make_shared(inode_count++);
>> -node0->add("meminfo", inode_count++, sysfs_meminfo);
>> +node0->add("meminfo", inode_count++, [] { return  
>> pseudofs::meminfo("Node 0 MemTotal:\t%ld kB\nNode 0 MemFree: \t%ld 
>> kB\n");  
>> });
>>   node0->add("cpumap", inode_count++, sysfs_cpumap);
>>   node0->add("distance", inode_count++, sysfs_distance);
>>
>> -- 
>> You received this message because you are subscribed to the Google Groups 
>> "OSv Development" group.
>> To unsubscribe from this group and stop receiving emails from it, send an 
>> email to osv...@googlegroups.com .
>> To view this discussion on the web visit 
>> https://groups.google.com/d/msgid/osv-dev/4b63eb059bc19023%40google.com
>> .
>>
>

-- 
You received this message because you are subscribed to the Google Groups "OSv 
Development" group.
To unsubscribe from this group and stop receiving emails from it, send an email 
to osv-dev+unsubscr...@googlegroups.com.
To view this discussion on the web visit 
https://groups.google.com/d/msgid/osv-dev/b0a08a05-d5d7-411b-993b-a012e63f615c%40googlegroups.com.

[osv-dev] [PATCH] elf: separate loading segments from processing headers

[Waldemar Kozaczuk]

The issue #1045 exposed a bug in the dynamic linker when processing
some headers like PT_NOTE require accessing data in the PT_LOAD segments.
If PT_NOTE headers come earlier than PT_NOTE we end up with the page fault
as the portions of ELF file have not been mapped yet. 

This patch separates loading of segments - load_segments() - from
processing headers by adding new method - process_headers().

Fixes #1045

Signed-off-by: Waldemar Kozaczuk 
---
 core/elf.cc| 17 ++---
 include/osv/elf.hh |  1 +
 2 files changed, 15 insertions(+), 3 deletions(-)

diff --git a/core/elf.cc b/core/elf.cc
index 39de219f..73834b19 100644
--- a/core/elf.cc
+++ b/core/elf.cc
@@ -452,14 +452,22 @@ extern "C" char _pie_static_tls_start, 
_pie_static_tls_end;
 void object::load_segments()
 {
 elf_debug("Loading segments\n");
+for (unsigned i = 0; i < _ehdr.e_phnum; ++i) {
+auto  = _phdrs[i];
+if (phdr.p_type == PT_LOAD) {
+load_segment(phdr);
+}
+}
+}
+
+void object::process_headers()
+{
+elf_debug("Processing headers\n");
 for (unsigned i = 0; i < _ehdr.e_phnum; ++i) {
 auto  = _phdrs[i];
 switch (phdr.p_type) {
 case PT_NULL:
 break;
-case PT_LOAD:
-load_segment(phdr);
-break;
 case PT_DYNAMIC:
 _dynamic_table = reinterpret_cast(_base + 
phdr.p_vaddr);
 break;
@@ -495,6 +503,7 @@ void object::load_segments()
 }
 break;
 }
+case PT_LOAD:
 case PT_PHDR:
 case PT_GNU_STACK:
 case PT_GNU_RELRO:
@@ -1233,6 +1242,7 @@ program::program(void* addr)
 _core->set_base(program_base);
 assert(_core->module_index() == core_module_index);
 _core->load_segments();
+_core->process_headers();
 set_search_path({"/", "/usr/lib"});
 // Our kernel already supplies the features of a bunch of traditional
 // shared libraries:
@@ -1361,6 +1371,7 @@ program::load_object(std::string name, 
std::vector extra_path,
 _modules_rcu.assign(new_modules.release());
 osv::rcu_dispose(old_modules);
 ef->load_segments();
+ef->process_headers();
 if (!ef->is_non_pie_executable())
_next_alloc = ef->end();
 add_debugger_obj(ef.get());
diff --git a/include/osv/elf.hh b/include/osv/elf.hh
index 3722da9e..1dac6c08 100644
--- a/include/osv/elf.hh
+++ b/include/osv/elf.hh
@@ -348,6 +348,7 @@ public:
 void* end() const;
 Elf64_Sym* lookup_symbol(const char* name, bool self_lookup);
 void load_segments();
+void process_headers();
 void unload_segments();
 void fix_permissions();
 void* resolve_pltgot(unsigned index);
-- 
2.20.1

-- 
You received this message because you are subscribed to the Google Groups "OSv 
Development" group.
To unsubscribe from this group and stop receiving emails from it, send an email 
to osv-dev+unsubscr...@googlegroups.com.
To view this discussion on the web visit 
https://groups.google.com/d/msgid/osv-dev/20200114125901.16800-1-jwkozaczuk%40gmail.com.

[osv-dev] [COMMIT osv master] stdio: fix scanf bug

[Commit Bot]

From: Nadav Har'El 
Committer: Waldemar Kozaczuk 
Branch: master

stdio: fix scanf bug

Lewis Tian noted on the mailing list that an application reading with
fscanf() from a large file fails after passing one BUFSIZ of data.

It turns out that the old Musl code we were using was only correct assuming
a certain bizarre readv() bug workaround which OSv dropped in commit
0b651428b91663255d8da9a4913663d0cd4cc710. But the Musl developers have
too noticed they can't rely on this workaround (which doesn't even apply
to unbuffered streams), and came up with a fix in their commit

https://git.musl-libc.org/cgit/musl/commit/src/internal/shgetc.c?id=c20804500deebaabc56f383d48dd1ac77dce8349

This patch does the same fix in our copy of the Musl code.
By the way, after this change libc/internal/shgetc.c is now identical to
musl/src/internal/shgetc.c but we can't drop the former yet because of some
header file differences that still need to be cleared up.

Here is the original Musl commit message describing the change:

  "fix major scanf breakage with unbuffered streams, fmemopen, etc.
  the shgetc api, used internally in scanf and int/float scanning code
  to handle field width limiting and pushback, was designed assuming
  that pushback could be achieved via a simple decrement on the file
  buffer pointer. this only worked by chance for regular FILE streams,
  due to the linux readv bug workaround in __stdio_read which moves the
  last requested byte through the buffer rather than directly back to
  the caller. for unbuffered streams and streams not using __stdio_read
  but some other underlying read function, the first character read
  could be completely lost, and replaced by whatever junk happened to be
  in the unget buffer.

  to fix this, simply have shgetc, when it performs an underlying read
  operation on the stream, store the character read at the -1 offset
  from the read buffer pointer. this is valid even for unbuffered
  streams, as they have an unget buffer located just below the start of
  the zero-length buffer. the check to avoid storing the character when
  it is already there is to handle the possibility of read-only buffers.
  no application-exposed FILE types are allowed to use read-only
  buffers, but sscanf and strto* may use them internally when calling
  functions which use the shgetc api."

Signed-off-by: Nadav Har'El 
Message-Id: <20200114111838.24313-1-...@scylladb.com>

---
diff --git a/libc/internal/shgetc.c b/libc/internal/shgetc.c
--- a/libc/internal/shgetc.c
+++ b/libc/internal/shgetc.c
@@ -22,5 +22,6 @@ int __shgetc(FILE *f)
else
f->shend = f->rend;
if (f->rend) f->shcnt += f->rend - f->rpos + 1;
+   if (f->rpos[-1] != c) f->rpos[-1] = c;
return c;
 }

--
You received this message because you are subscribed to the Google Groups "OSv 
Development" group.
To unsubscribe from this group and stop receiving emails from it, send an email 
to osv-dev+unsubscr...@googlegroups.com.
To view this discussion on the web visit 
https://groups.google.com/d/msgid/osv-dev/e7d91b059c18c95a%40google.com.

[osv-dev] [COMMIT osv master] git: ignore JSON Compilation Database files used by CLion IDE

[Commit Bot]

From: Waldemar Kozaczuk 
Committer: Waldemar Kozaczuk 
Branch: master

git: ignore JSON Compilation Database files used by CLion IDE

Signed-off-by: Waldemar Kozaczuk 

---
diff --git a/.gitignore b/.gitignore
--- a/.gitignore
+++ b/.gitignore
@@ -49,3 +49,4 @@ modules/libyaml/usr.manifest
 modules/dl_tests/bionic/
 modules/dl_tests/usr.manifest
 .idea
+compile_commands.json

--
You received this message because you are subscribed to the Google Groups "OSv 
Development" group.
To unsubscribe from this group and stop receiving emails from it, send an email 
to osv-dev+unsubscr...@googlegroups.com.
To view this discussion on the web visit 
https://groups.google.com/d/msgid/osv-dev/fa746a059c18c944%40google.com.

[osv-dev] [PATCH] stdio: fix scanf bug

[Nadav Har'El]

Lewis Tian noted on the mailing list that an application reading with
fscanf() from a large file fails after passing one BUFSIZ of data.

It turns out that the old Musl code we were using was only correct assuming
a certain bizarre readv() bug workaround which OSv dropped in commit
0b651428b91663255d8da9a4913663d0cd4cc710. But the Musl developers have
too noticed they can't rely on this workaround (which doesn't even apply
to unbuffered streams), and came up with a fix in their commit

https://git.musl-libc.org/cgit/musl/commit/src/internal/shgetc.c?id=c20804500deebaabc56f383d48dd1ac77dce8349

This patch does the same fix in our copy of the Musl code.
By the way, after this change libc/internal/shgetc.c is now identical to
musl/src/internal/shgetc.c but we can't drop the former yet because of some
header file differences that still need to be cleared up.

Here is the original Musl commit message describing the change:

  "fix major scanf breakage with unbuffered streams, fmemopen, etc.
  the shgetc api, used internally in scanf and int/float scanning code
  to handle field width limiting and pushback, was designed assuming
  that pushback could be achieved via a simple decrement on the file
  buffer pointer. this only worked by chance for regular FILE streams,
  due to the linux readv bug workaround in __stdio_read which moves the
  last requested byte through the buffer rather than directly back to
  the caller. for unbuffered streams and streams not using __stdio_read
  but some other underlying read function, the first character read
  could be completely lost, and replaced by whatever junk happened to be
  in the unget buffer.

  to fix this, simply have shgetc, when it performs an underlying read
  operation on the stream, store the character read at the -1 offset
  from the read buffer pointer. this is valid even for unbuffered
  streams, as they have an unget buffer located just below the start of
  the zero-length buffer. the check to avoid storing the character when
  it is already there is to handle the possibility of read-only buffers.
  no application-exposed FILE types are allowed to use read-only
  buffers, but sscanf and strto* may use them internally when calling
  functions which use the shgetc api."

Signed-off-by: Nadav Har'El 
---
 libc/internal/shgetc.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/libc/internal/shgetc.c b/libc/internal/shgetc.c
index 0641c835..5a47ac9e 100644
--- a/libc/internal/shgetc.c
+++ b/libc/internal/shgetc.c
@@ -22,5 +22,6 @@ int __shgetc(FILE *f)
else
f->shend = f->rend;
if (f->rend) f->shcnt += f->rend - f->rpos + 1;
+   if (f->rpos[-1] != c) f->rpos[-1] = c;
return c;
 }
-- 
2.21.1

-- 
You received this message because you are subscribed to the Google Groups "OSv 
Development" group.
To unsubscribe from this group and stop receiving emails from it, send an email 
to osv-dev+unsubscr...@googlegroups.com.
To view this discussion on the web visit 
https://groups.google.com/d/msgid/osv-dev/20200114111838.24313-1-nyh%40scylladb.com.

Re: [osv-dev] Fail to read data from large file

[Nadav Har'El]

On Tue, Jan 14, 2020 at 12:41 PM Lewis Tian  wrote:

>
>
> On Tuesday, January 14, 2020 at 4:45:27 PM UTC+8, Nadav Har'El wrote:
>>
>>
>> Interesting. Smells like a serious stdio bug that needs to be debugged :-(
>> I think it's not a coincidence that position is 1025, with stdio's
>> BUFSIZ=1024.
>>
>> Just as a completely wild guess, can you please try if the following
>> patch to libc/internal/shgetc.c helps?
>>
>> @@ -22,5 +22,6 @@
>>   else
>>   f->shend = f->rend;
>>   if (f->rend) f->shcnt += f->rend - f->rpos + 1;
>> + if (f->rpos[-1] != c) f->rpos[-1] = c;
>>   return c;
>>  }
>>
>>
> Yeah, the patch is helpful! Thanks! : )
>

I'm pleasantly surprised :-) I'll send a more organized patch to the
mailing list.

>
>
>>
>>>
>>>


> When I use a small graph (4 nodes, 7 edges), it runs normally.
>
> taseikyo@ubuntu:~/Desktop/osv$ ./scripts/build image=my-pagerank-test
> taseikyo@ubuntu:~/Desktop/osv$ ./scripts/run.py
> OSv v0.54.0-71-g69a0ce39
> eth0: 192.168.122.15
> Booted up in 356.44 ms
> Cmdline: /pagerank
>
> Graph data:
>
>   Nodes: 4, Edges: 7
>
>
> Number of iteration to converge: 41
>
> Final Pagerank values:
>
> [0.159913 , 0.144016 , 0.144016 , 0.082809 ]
>
> Time spent: 0.693802 seconds.
>
> Is osv unable to read large files (bug?) I'll appreciate your help
> very much! : )
>
> --
> You received this message because you are subscribed to the Google
> Groups "OSv Development" group.
> To unsubscribe from this group and stop receiving emails from it, send
> an email to osv...@googlegroups.com.
> To view this discussion on the web visit
> https://groups.google.com/d/msgid/osv-dev/50f3a073-e364-484c-803e-570e0dd6530c%40googlegroups.com
> 
> .
>
 --
>>> You received this message because you are subscribed to the Google
>>> Groups "OSv Development" group.
>>> To unsubscribe from this group and stop receiving emails from it, send
>>> an email to osv...@googlegroups.com.
>>> To view this discussion on the web visit
>>> https://groups.google.com/d/msgid/osv-dev/ae3d8e59-bb13-40a4-bdd0-b03ec89b9ff3%40googlegroups.com
>>> 
>>> .
>>>
>> --
> You received this message because you are subscribed to the Google Groups
> "OSv Development" group.
> To unsubscribe from this group and stop receiving emails from it, send an
> email to osv-dev+unsubscr...@googlegroups.com.
> To view this discussion on the web visit
> https://groups.google.com/d/msgid/osv-dev/9e83516e-d25f-4409-a15d-a0a59871978d%40googlegroups.com
> 
> .
>

-- 
You received this message because you are subscribed to the Google Groups "OSv 
Development" group.
To unsubscribe from this group and stop receiving emails from it, send an email 
to osv-dev+unsubscr...@googlegroups.com.
To view this discussion on the web visit 
https://groups.google.com/d/msgid/osv-dev/CANEVyjva7W44mktoixKtdprxbEyXNt1qg7%3D%3D6qNatcH3xAGAfA%40mail.gmail.com.

Re: [osv-dev] Fail to read data from large file

[Lewis Tian]

On Tuesday, January 14, 2020 at 4:45:27 PM UTC+8, Nadav Har'El wrote:
>
>
> On Tue, Jan 14, 2020 at 10:15 AM Lewis Tian  > wrote:
>
>>
>>
>> On Tuesday, January 14, 2020 at 3:45:09 PM UTC+8, Nadav Har'El wrote:
>>>
>>>
>>> On Tue, Jan 14, 2020 at 9:28 AM Lewis Tian  wrote:
>>>
 When I run pagerank on Ubuntu, the code works fine. (the graph data is 
 stored in web-NotreDame.txt, 21M)

>>>
>>> 21M isn't very large, it shouldn't present any special problems. We 
>>> probably have a bug that isn't just about file size:
>>>
>>>
 taseikyo@ubuntu:~/Desktop/osv/apps/my-pagerank-test$ ls
 Makefile  module.py  pagerank.c  usr.manifest  web-NotreDame.txt

 taseikyo@ubuntu:~/Desktop/osv/apps/my-pagerank-test$ make
 cc -pie -o pagerank  pagerank.c

 taseikyo@ubuntu:~/Desktop/osv/apps/my-pagerank-test$ ll -h
 total 21M
 drwxrwxr-x   2 taseikyo taseikyo 4.0K Jan 14 11:59 ./
 drwxrwxr-x 128 taseikyo taseikyo 4.0K Jan 14 11:51 ../
 -rw-rw-r--   1 taseikyo taseikyo  110 Jan 14 11:51 Makefile
 -rw-rw-r--   1 taseikyo taseikyo   60 Jan 13 11:26 module.py
 -rwxrwxr-x   1 taseikyo taseikyo  17K Jan 14 11:59 pagerank*
 -rw-rw-r--   1 taseikyo taseikyo 5.6K Jan 14 11:51 pagerank.c
 -rw-rw-r--   1 taseikyo taseikyo   85 Jan 14 11:53 usr.manifest
 -rw-rw-r--   1 taseikyo taseikyo  21M Jan 14 11:54 web-NotreDame.txt

 taseikyo@ubuntu:~/Desktop/osv/apps/my-pagerank-test$ ./pagerank
 Graph data:

   Nodes: 325729, Edges: 1497134 


 Number of iteration to converge: 52 

 Final Pagerank values:
 [0.002066 , 0.000181 , ...]

 Time spent: 0.896491 seconds.

 But when I build and run pagerank on osv, it fails to read the graph 
 data (only read part of the graph).

 taseikyo@ubuntu:~/Desktop/osv$ ./scripts/build image=my-pagerank-test
 taseikyo@ubuntu:~/Desktop/osv$ ./scripts/run.py
 OSv v0.54.0-71-g69a0ce39
 eth0: 192.168.122.15
 Booted up in 338.86 ms
 Cmdline: /pagerank

 Graph data:

   Nodes: 325729, Edges: 1497134 

 Fail to read data...

 From: 6 To: 119

 Here is part of the code:

 while (!feof(fp)) {
 fret = fscanf(fp, "%d%d", , );
 if (fret == 0) {
 printf("Fail to read data...\n");
 printf("\n From: %d To: %d\n",fromnode, tonode);
 return -1;
 }
 ...
 }


>>> Maybe we have a bug in fscanf or in the stdio reading layer?
>>> Things I'd like you to please check:
>>> 1. Print a failure already if fret < 2 - since 1 is also a failure.
>>> 2. On failure, please print ftell(fp) - our position in the file (is it 
>>> the end? something in the middle?). Please also do a fgets() or a short 
>>> fgetc() loop or fread() to read the next available bytes, to try to 
>>> understand by fscanf() failed. Is the reading from the file failing, or is 
>>> the parsing failing?
>>>
>>>
>> Thanks for your advice,
>> On failure, the output is as follows:
>>
>> Graph data:
>>
>>   Nodes: 325729, Edges: 1497134 
>>
>> fret: 1
>> position: 1025
>> next char: 
>>
>> fret is 1, fp is in the middle, fgetc or fgets cannot read anything. I 
>> think it should be the former case.
>>
>
> Interesting. Smells like a serious stdio bug that needs to be debugged :-(
> I think it's not a coincidence that position is 1025, with stdio's 
> BUFSIZ=1024.
>
> Just as a completely wild guess, can you please try if the following patch 
> to libc/internal/shgetc.c helps?
>
> @@ -22,5 +22,6 @@
>   else
>   f->shend = f->rend;
>   if (f->rend) f->shcnt += f->rend - f->rpos + 1;
> + if (f->rpos[-1] != c) f->rpos[-1] = c;
>   return c;
>  }
>
>
Yeah, the patch is helpful! Thanks! : )
 

>
>>  
>>
>>>  
>>>
 When I use a small graph (4 nodes, 7 edges), it runs normally.

 taseikyo@ubuntu:~/Desktop/osv$ ./scripts/build image=my-pagerank-test
 taseikyo@ubuntu:~/Desktop/osv$ ./scripts/run.py
 OSv v0.54.0-71-g69a0ce39
 eth0: 192.168.122.15
 Booted up in 356.44 ms
 Cmdline: /pagerank

 Graph data:

   Nodes: 4, Edges: 7 


 Number of iteration to converge: 41 

 Final Pagerank values:

 [0.159913 , 0.144016 , 0.144016 , 0.082809 ]

 Time spent: 0.693802 seconds.

 Is osv unable to read large files (bug?) I'll appreciate your help very 
 much! : )

 -- 
 You received this message because you are subscribed to the Google 
 Groups "OSv Development" group.
 To unsubscribe from this group and stop receiving emails from it, send 
 an email to osv...@googlegroups.com.
 To view this discussion on the web visit 
 https://groups.google.com/d/msgid/osv-dev/50f3a073-e364-484c-803e-570e0dd6530c%40googlegroups.com
  
 
 .

>>> -- 
>> You received this message because you are 

Re: [osv-dev] Fail to read data from large file

[Nadav Har'El]

On Tue, Jan 14, 2020 at 10:15 AM Lewis Tian  wrote:

>
>
> On Tuesday, January 14, 2020 at 3:45:09 PM UTC+8, Nadav Har'El wrote:
>>
>>
>> On Tue, Jan 14, 2020 at 9:28 AM Lewis Tian  wrote:
>>
>>> When I run pagerank on Ubuntu, the code works fine. (the graph data is
>>> stored in web-NotreDame.txt, 21M)
>>>
>>
>> 21M isn't very large, it shouldn't present any special problems. We
>> probably have a bug that isn't just about file size:
>>
>>
>>> taseikyo@ubuntu:~/Desktop/osv/apps/my-pagerank-test$ ls
>>> Makefile  module.py  pagerank.c  usr.manifest  web-NotreDame.txt
>>>
>>> taseikyo@ubuntu:~/Desktop/osv/apps/my-pagerank-test$ make
>>> cc -pie -o pagerank  pagerank.c
>>>
>>> taseikyo@ubuntu:~/Desktop/osv/apps/my-pagerank-test$ ll -h
>>> total 21M
>>> drwxrwxr-x   2 taseikyo taseikyo 4.0K Jan 14 11:59 ./
>>> drwxrwxr-x 128 taseikyo taseikyo 4.0K Jan 14 11:51 ../
>>> -rw-rw-r--   1 taseikyo taseikyo  110 Jan 14 11:51 Makefile
>>> -rw-rw-r--   1 taseikyo taseikyo   60 Jan 13 11:26 module.py
>>> -rwxrwxr-x   1 taseikyo taseikyo  17K Jan 14 11:59 pagerank*
>>> -rw-rw-r--   1 taseikyo taseikyo 5.6K Jan 14 11:51 pagerank.c
>>> -rw-rw-r--   1 taseikyo taseikyo   85 Jan 14 11:53 usr.manifest
>>> -rw-rw-r--   1 taseikyo taseikyo  21M Jan 14 11:54 web-NotreDame.txt
>>>
>>> taseikyo@ubuntu:~/Desktop/osv/apps/my-pagerank-test$ ./pagerank
>>> Graph data:
>>>
>>>   Nodes: 325729, Edges: 1497134
>>>
>>>
>>> Number of iteration to converge: 52
>>>
>>> Final Pagerank values:
>>> [0.002066 , 0.000181 , ...]
>>>
>>> Time spent: 0.896491 seconds.
>>>
>>> But when I build and run pagerank on osv, it fails to read the graph
>>> data (only read part of the graph).
>>>
>>> taseikyo@ubuntu:~/Desktop/osv$ ./scripts/build image=my-pagerank-test
>>> taseikyo@ubuntu:~/Desktop/osv$ ./scripts/run.py
>>> OSv v0.54.0-71-g69a0ce39
>>> eth0: 192.168.122.15
>>> Booted up in 338.86 ms
>>> Cmdline: /pagerank
>>>
>>> Graph data:
>>>
>>>   Nodes: 325729, Edges: 1497134
>>>
>>> Fail to read data...
>>>
>>> From: 6 To: 119
>>>
>>> Here is part of the code:
>>>
>>> while (!feof(fp)) {
>>> fret = fscanf(fp, "%d%d", , );
>>> if (fret == 0) {
>>> printf("Fail to read data...\n");
>>> printf("\n From: %d To: %d\n",fromnode, tonode);
>>> return -1;
>>> }
>>> ...
>>> }
>>>
>>>
>> Maybe we have a bug in fscanf or in the stdio reading layer?
>> Things I'd like you to please check:
>> 1. Print a failure already if fret < 2 - since 1 is also a failure.
>> 2. On failure, please print ftell(fp) - our position in the file (is it
>> the end? something in the middle?). Please also do a fgets() or a short
>> fgetc() loop or fread() to read the next available bytes, to try to
>> understand by fscanf() failed. Is the reading from the file failing, or is
>> the parsing failing?
>>
>>
> Thanks for your advice,
> On failure, the output is as follows:
>
> Graph data:
>
>   Nodes: 325729, Edges: 1497134
>
> fret: 1
> position: 1025
> next char:
>
> fret is 1, fp is in the middle, fgetc or fgets cannot read anything. I
> think it should be the former case.
>

Interesting. Smells like a serious stdio bug that needs to be debugged :-(
I think it's not a coincidence that position is 1025, with stdio's
BUFSIZ=1024.

Just as a completely wild guess, can you please try if the following patch
to libc/internal/shgetc.c helps?

@@ -22,5 +22,6 @@
  else
  f->shend = f->rend;
  if (f->rend) f->shcnt += f->rend - f->rpos + 1;
+ if (f->rpos[-1] != c) f->rpos[-1] = c;
  return c;
 }



>
>
>>
>>
>>> When I use a small graph (4 nodes, 7 edges), it runs normally.
>>>
>>> taseikyo@ubuntu:~/Desktop/osv$ ./scripts/build image=my-pagerank-test
>>> taseikyo@ubuntu:~/Desktop/osv$ ./scripts/run.py
>>> OSv v0.54.0-71-g69a0ce39
>>> eth0: 192.168.122.15
>>> Booted up in 356.44 ms
>>> Cmdline: /pagerank
>>>
>>> Graph data:
>>>
>>>   Nodes: 4, Edges: 7
>>>
>>>
>>> Number of iteration to converge: 41
>>>
>>> Final Pagerank values:
>>>
>>> [0.159913 , 0.144016 , 0.144016 , 0.082809 ]
>>>
>>> Time spent: 0.693802 seconds.
>>>
>>> Is osv unable to read large files (bug?) I'll appreciate your help very
>>> much! : )
>>>
>>> --
>>> You received this message because you are subscribed to the Google
>>> Groups "OSv Development" group.
>>> To unsubscribe from this group and stop receiving emails from it, send
>>> an email to osv...@googlegroups.com.
>>> To view this discussion on the web visit
>>> https://groups.google.com/d/msgid/osv-dev/50f3a073-e364-484c-803e-570e0dd6530c%40googlegroups.com
>>> 
>>> .
>>>
>> --
> You received this message because you are subscribed to the Google Groups
> "OSv Development" group.
> To unsubscribe from this group and stop receiving emails from it, send an
> email to osv-dev+unsubscr...@googlegroups.com.
> To view this discussion on the web visit
> https://groups.google.com/d/msgid/osv-dev/ae3d8e59-bb13-40a4-bdd0-b03ec89b9ff3%40googlegroups.com
> 

Re: [osv-dev] Fail to read data from large file

[Lewis Tian]

On Tuesday, January 14, 2020 at 3:45:09 PM UTC+8, Nadav Har'El wrote:
>
>
> On Tue, Jan 14, 2020 at 9:28 AM Lewis Tian  > wrote:
>
>> When I run pagerank on Ubuntu, the code works fine. (the graph data is 
>> stored in web-NotreDame.txt, 21M)
>>
>
> 21M isn't very large, it shouldn't present any special problems. We 
> probably have a bug that isn't just about file size:
>
>
>> taseikyo@ubuntu:~/Desktop/osv/apps/my-pagerank-test$ ls
>> Makefile  module.py  pagerank.c  usr.manifest  web-NotreDame.txt
>>
>> taseikyo@ubuntu:~/Desktop/osv/apps/my-pagerank-test$ make
>> cc -pie -o pagerank  pagerank.c
>>
>> taseikyo@ubuntu:~/Desktop/osv/apps/my-pagerank-test$ ll -h
>> total 21M
>> drwxrwxr-x   2 taseikyo taseikyo 4.0K Jan 14 11:59 ./
>> drwxrwxr-x 128 taseikyo taseikyo 4.0K Jan 14 11:51 ../
>> -rw-rw-r--   1 taseikyo taseikyo  110 Jan 14 11:51 Makefile
>> -rw-rw-r--   1 taseikyo taseikyo   60 Jan 13 11:26 module.py
>> -rwxrwxr-x   1 taseikyo taseikyo  17K Jan 14 11:59 pagerank*
>> -rw-rw-r--   1 taseikyo taseikyo 5.6K Jan 14 11:51 pagerank.c
>> -rw-rw-r--   1 taseikyo taseikyo   85 Jan 14 11:53 usr.manifest
>> -rw-rw-r--   1 taseikyo taseikyo  21M Jan 14 11:54 web-NotreDame.txt
>>
>> taseikyo@ubuntu:~/Desktop/osv/apps/my-pagerank-test$ ./pagerank
>> Graph data:
>>
>>   Nodes: 325729, Edges: 1497134 
>>
>>
>> Number of iteration to converge: 52 
>>
>> Final Pagerank values:
>> [0.002066 , 0.000181 , ...]
>>
>> Time spent: 0.896491 seconds.
>>
>> But when I build and run pagerank on osv, it fails to read the graph data 
>> (only read part of the graph).
>>
>> taseikyo@ubuntu:~/Desktop/osv$ ./scripts/build image=my-pagerank-test
>> taseikyo@ubuntu:~/Desktop/osv$ ./scripts/run.py
>> OSv v0.54.0-71-g69a0ce39
>> eth0: 192.168.122.15
>> Booted up in 338.86 ms
>> Cmdline: /pagerank
>>
>> Graph data:
>>
>>   Nodes: 325729, Edges: 1497134 
>>
>> Fail to read data...
>>
>> From: 6 To: 119
>>
>> Here is part of the code:
>>
>> while (!feof(fp)) {
>> fret = fscanf(fp, "%d%d", , );
>> if (fret == 0) {
>> printf("Fail to read data...\n");
>> printf("\n From: %d To: %d\n",fromnode, tonode);
>> return -1;
>> }
>> ...
>> }
>>
>>
> Maybe we have a bug in fscanf or in the stdio reading layer?
> Things I'd like you to please check:
> 1. Print a failure already if fret < 2 - since 1 is also a failure.
> 2. On failure, please print ftell(fp) - our position in the file (is it 
> the end? something in the middle?). Please also do a fgets() or a short 
> fgetc() loop or fread() to read the next available bytes, to try to 
> understand by fscanf() failed. Is the reading from the file failing, or is 
> the parsing failing?
>
>
Thanks for your advice,
On failure, the output is as follows:

Graph data:

  Nodes: 325729, Edges: 1497134 

fret: 1
position: 1025
next char: 

fret is 1, fp is in the middle, fgetc or fgets cannot read anything. I 
think it should be the former case.

 

>  
>
>> When I use a small graph (4 nodes, 7 edges), it runs normally.
>>
>> taseikyo@ubuntu:~/Desktop/osv$ ./scripts/build image=my-pagerank-test
>> taseikyo@ubuntu:~/Desktop/osv$ ./scripts/run.py
>> OSv v0.54.0-71-g69a0ce39
>> eth0: 192.168.122.15
>> Booted up in 356.44 ms
>> Cmdline: /pagerank
>>
>> Graph data:
>>
>>   Nodes: 4, Edges: 7 
>>
>>
>> Number of iteration to converge: 41 
>>
>> Final Pagerank values:
>>
>> [0.159913 , 0.144016 , 0.144016 , 0.082809 ]
>>
>> Time spent: 0.693802 seconds.
>>
>> Is osv unable to read large files (bug?) I'll appreciate your help very 
>> much! : )
>>
>> -- 
>> You received this message because you are subscribed to the Google Groups 
>> "OSv Development" group.
>> To unsubscribe from this group and stop receiving emails from it, send an 
>> email to osv...@googlegroups.com .
>> To view this discussion on the web visit 
>> https://groups.google.com/d/msgid/osv-dev/50f3a073-e364-484c-803e-570e0dd6530c%40googlegroups.com
>>  
>> 
>> .
>>
>

-- 
You received this message because you are subscribed to the Google Groups "OSv 
Development" group.
To unsubscribe from this group and stop receiving emails from it, send an email 
to osv-dev+unsubscr...@googlegroups.com.
To view this discussion on the web visit 
https://groups.google.com/d/msgid/osv-dev/ae3d8e59-bb13-40a4-bdd0-b03ec89b9ff3%40googlegroups.com.