Playing with py.execnet failes too. Full traceback attached.
P.S. Seens like a hard start with my trials in PyPy =)
Armin Rigo wrote:
Hi,
On Thu, Aug 28, 2008 at 10:34:51PM +0600, Vetoshkin Nikita wrote:
Is there a direction, where I could digg to help? Something to begin with.
I don't have enough experience but I'd like to take part (even very
small) in this project.
Sure. Do you understand the issue? It's related to the way a Unix
process starts a new subprocess: os.fork() followed by os.execv() in the
child process. The issue is that os.fork() fails if the parent process'
total memory is more than half of your RAM+swap, because it tries to
create a copy of the process, and the copy would also need more than
half of your RAM+swap. (In reality, os.fork() doesn't copy the memory
at all, but creates "shared pages" of memory so that memory pages are
duplicated only when one of the two processes really modifies it; but
still os.fork() has to mark all the memory as *potentially* used, and
raises MemoryError if there isn't enough.)
A solution might be to split translate.py in two processes: translate.py
would start a subprocess when it starts, and then do all the work
(consuming a lot of RAM); but when it needs to start new processes, e.g.
call gcc, it would instead ask the subprocess to start the new
processes. More precisely, the code in pypy.translator.tool.cbuild
would stop using distutils directly, and instead ask the subprocess to
use distutils. It looks even easy to do with the help of a
py.execnet.PopenGateway() created when translate.py starts (see the py
lib documentation at http://codespeak.net/py/).
A bientot,
Armin
--
Nikita
>>>> gw = py.execnet.PopenGateway()
faking <type '_socket.socket'>
faking <class 'socket.timeout'>
Traceback (most recent call last):
File "./pypy/bin/py.py", line 152, in <module>
sys.exit(main_(sys.argv))
File "./pypy/bin/py.py", line 138, in main_
con.interact(banner)
File "/var/tmp/pypy-svn/dist/pypy/interpreter/interactive.py", line
133, in interact
code.InteractiveConsole.interact(self, banner)
File "/usr/lib64/python2.5/code.py", line 239, in interact
more = self.push(line)
File "/usr/lib64/python2.5/code.py", line 261, in push
more = self.runsource(source, self.filename)
File "/var/tmp/pypy-svn/dist/pypy/interpreter/interactive.py", line
192, in runsource
main.run_toplevel(self.space, doit, verbose=self.verbose)
File "/var/tmp/pypy-svn/dist/pypy/interpreter/main.py", line 103, in
run_toplevel
f()
File "/var/tmp/pypy-svn/dist/pypy/interpreter/interactive.py", line
184, in doit
code.exec_code(self.space, self.w_globals, self.w_globals)
File "/var/tmp/pypy-svn/dist/pypy/interpreter/eval.py", line 27, in
exec_code
return frame.run()
File "/var/tmp/pypy-svn/dist/pypy/interpreter/pyframe.py", line 91, in
run
return self.execute_frame()
File "/var/tmp/pypy-svn/dist/pypy/interpreter/pyframe.py", line 117,
in execute_frame
executioncontext)
File "/var/tmp/pypy-svn/dist/pypy/interpreter/pyopcode.py", line 79,
in dispatch
next_instr = self.handle_bytecode(co_code, next_instr, ec)
File "/var/tmp/pypy-svn/dist/pypy/interpreter/pyopcode.py", line 89,
in handle_bytecode
next_instr = self.dispatch_bytecode(co_code, next_instr, ec)
File "/var/tmp/pypy-svn/dist/pypy/interpreter/pyopcode.py", line 235,
in dispatch_bytecode
res = getattr(self, methodname)(oparg, next_instr)
File "/var/tmp/pypy-svn/dist/pypy/interpreter/pyopcode.py", line 695,
in LOAD_ATTR
w_value = f.space.getattr(w_obj, w_attributename)
File "/var/tmp/pypy-svn/dist/pypy/objspace/std/objspace.py", line 669,
in getattr
return DescrOperation.getattr(self, w_obj, w_name)
File "/var/tmp/pypy-svn/dist/pypy/objspace/descroperation.py", line
134, in getattr
return space._handle_getattribute(w_descr, w_obj, w_name)
File "/var/tmp/pypy-svn/dist/pypy/objspace/descroperation.py", line
147, in _handle_getattribute
return space.get_and_call_function(w_descr, w_obj, w_name)
File "/var/tmp/pypy-svn/dist/pypy/objspace/descroperation.py", line
89, in get_and_call_function
return descr.funccall(w_obj, *args_w)
File "/var/tmp/pypy-svn/dist/pypy/interpreter/function.py", line 56,
in funccall
return code.fastcall_2(self.space, self, args_w[0], args_w[1])
File "/var/tmp/pypy-svn/dist/pypy/interpreter/pycode.py", line 191, in
fastcall_2
return frame.run()
File "/var/tmp/pypy-svn/dist/pypy/interpreter/pyframe.py", line 91, in
run
return self.execute_frame()
File "/var/tmp/pypy-svn/dist/pypy/interpreter/pyframe.py", line 117,
in execute_frame
executioncontext)
File "/var/tmp/pypy-svn/dist/pypy/interpreter/pyopcode.py", line 79,
in dispatch
next_instr = self.handle_bytecode(co_code, next_instr, ec)
File "/var/tmp/pypy-svn/dist/pypy/interpreter/pyopcode.py", line 89,
in handle_bytecode
next_instr = self.dispatch_bytecode(co_code, next_instr, ec)
File "/var/tmp/pypy-svn/dist/pypy/interpreter/pyopcode.py", line 235,
in dispatch_bytecode
res = getattr(self, methodname)(oparg, next_instr)
File "/var/tmp/pypy-svn/dist/pypy/interpreter/pyopcode.py", line 881,
in CALL_FUNCTION
w_result = f.space.call_valuestack(w_function, nargs, f)
File "/var/tmp/pypy-svn/dist/pypy/interpreter/baseobjspace.py", line
736, in call_valuestack
return w_func.funccall_valuestack(nargs, frame)
File "/var/tmp/pypy-svn/dist/pypy/interpreter/function.py", line 81,
in funccall_valuestack
frame.peekvalue(0))
File "/var/tmp/pypy-svn/dist/pypy/interpreter/pycode.py", line 191, in
fastcall_2
return frame.run()
File "/var/tmp/pypy-svn/dist/pypy/interpreter/pyframe.py", line 91, in
run
return self.execute_frame()
File "/var/tmp/pypy-svn/dist/pypy/interpreter/pyframe.py", line 117,
in execute_frame
executioncontext)
File "/var/tmp/pypy-svn/dist/pypy/interpreter/pyopcode.py", line 79,
in dispatch
next_instr = self.handle_bytecode(co_code, next_instr, ec)
File "/var/tmp/pypy-svn/dist/pypy/interpreter/pyopcode.py", line 89,
in handle_bytecode
next_instr = self.dispatch_bytecode(co_code, next_instr, ec)
File "/var/tmp/pypy-svn/dist/pypy/interpreter/pyopcode.py", line 235,
in dispatch_bytecode
res = getattr(self, methodname)(oparg, next_instr)
File "/var/tmp/pypy-svn/dist/pypy/interpreter/pyopcode.py", line 881,
in CALL_FUNCTION
w_result = f.space.call_valuestack(w_function, nargs, f)
File "/var/tmp/pypy-svn/dist/pypy/interpreter/baseobjspace.py", line
736, in call_valuestack
return w_func.funccall_valuestack(nargs, frame)
File "/var/tmp/pypy-svn/dist/pypy/interpreter/function.py", line 81,
in funccall_valuestack
frame.peekvalue(0))
File "/var/tmp/pypy-svn/dist/pypy/interpreter/pycode.py", line 191, in
fastcall_2
return frame.run()
File "/var/tmp/pypy-svn/dist/pypy/interpreter/pyframe.py", line 91, in
run
return self.execute_frame()
File "/var/tmp/pypy-svn/dist/pypy/interpreter/pyframe.py", line 117,
in execute_frame
executioncontext)
File "/var/tmp/pypy-svn/dist/pypy/interpreter/pyopcode.py", line 79,
in dispatch
next_instr = self.handle_bytecode(co_code, next_instr, ec)
File "/var/tmp/pypy-svn/dist/pypy/interpreter/pyopcode.py", line 89,
in handle_bytecode
next_instr = self.dispatch_bytecode(co_code, next_instr, ec)
File "/var/tmp/pypy-svn/dist/pypy/interpreter/pyopcode.py", line 235,
in dispatch_bytecode
res = getattr(self, methodname)(oparg, next_instr)
File "/var/tmp/pypy-svn/dist/pypy/interpreter/pyopcode.py", line 881,
in CALL_FUNCTION
w_result = f.space.call_valuestack(w_function, nargs, f)
File "/var/tmp/pypy-svn/dist/pypy/interpreter/baseobjspace.py", line
736, in call_valuestack
return w_func.funccall_valuestack(nargs, frame)
File "/var/tmp/pypy-svn/dist/pypy/interpreter/function.py", line 88,
in funccall_valuestack
frame.peekvalue(0))
File "/var/tmp/pypy-svn/dist/pypy/interpreter/gateway.py", line 663,
in fastcall_4
w_result = self.fastfunc_4(space, w1, w2, w3, w4)
File "</var/tmp/pypy-svn/dist/py/code/source.py:213>", line 3, in
fastfunc_importhook_4
File "/var/tmp/pypy-svn/dist/pypy/module/__builtin__/importing.py",
line 180, in importhook
w_mod = absolute_import(space, modulename, 0, w_fromlist,
tentative=0)
File "/var/tmp/pypy-svn/dist/pypy/module/__builtin__/importing.py",
line 192, in absolute_import
w_fromlist, tentative)
File "/var/tmp/pypy-svn/dist/pypy/module/__builtin__/importing.py",
line 211, in _absolute_import
tentative=tentative)
File "/var/tmp/pypy-svn/dist/pypy/module/__builtin__/importing.py",
line 272, in load_part
w(partname))
File "/var/tmp/pypy-svn/dist/pypy/module/__builtin__/importing.py",
line 90, in try_import_mod
magic, timestamp, stream.readall())
File "/var/tmp/pypy-svn/dist/pypy/module/__builtin__/importing.py",
line 499, in load_compiled_module
code_w.exec_code(space, w_dic, w_dic)
File "/var/tmp/pypy-svn/dist/pypy/interpreter/eval.py", line 27, in
exec_code
return frame.run()
File "/var/tmp/pypy-svn/dist/pypy/interpreter/pyframe.py", line 91, in
run
return self.execute_frame()
File "/var/tmp/pypy-svn/dist/pypy/interpreter/pyframe.py", line 117,
in execute_frame
executioncontext)
File "/var/tmp/pypy-svn/dist/pypy/interpreter/pyopcode.py", line 79,
in dispatch
next_instr = self.handle_bytecode(co_code, next_instr, ec)
File "/var/tmp/pypy-svn/dist/pypy/interpreter/pyopcode.py", line 89,
in handle_bytecode
next_instr = self.dispatch_bytecode(co_code, next_instr, ec)
File "/var/tmp/pypy-svn/dist/pypy/interpreter/pyopcode.py", line 235,
in dispatch_bytecode
res = getattr(self, methodname)(oparg, next_instr)
File "/var/tmp/pypy-svn/dist/pypy/interpreter/pyopcode.py", line 760,
in IMPORT_NAME
w_locals, w_fromlist)
File "/var/tmp/pypy-svn/dist/pypy/interpreter/baseobjspace.py", line
712, in call_function
return w_func.funccall(*args_w)
File "/var/tmp/pypy-svn/dist/pypy/interpreter/function.py", line 62,
in funccall
args_w[1], args_w[2], args_w[3])
File "/var/tmp/pypy-svn/dist/pypy/interpreter/gateway.py", line 663,
in fastcall_4
w_result = self.fastfunc_4(space, w1, w2, w3, w4)
File "</var/tmp/pypy-svn/dist/py/code/source.py:213>", line 3, in
fastfunc_importhook_4
File "/var/tmp/pypy-svn/dist/pypy/module/__builtin__/importing.py",
line 180, in importhook
w_mod = absolute_import(space, modulename, 0, w_fromlist,
tentative=0)
File "/var/tmp/pypy-svn/dist/pypy/module/__builtin__/importing.py",
line 192, in absolute_import
w_fromlist, tentative)
File "/var/tmp/pypy-svn/dist/pypy/module/__builtin__/importing.py",
line 211, in _absolute_import
tentative=tentative)
File "/var/tmp/pypy-svn/dist/pypy/module/__builtin__/importing.py",
line 272, in load_part
w(partname))
File "/var/tmp/pypy-svn/dist/pypy/module/__builtin__/importing.py",
line 90, in try_import_mod
magic, timestamp, stream.readall())
File "/var/tmp/pypy-svn/dist/pypy/module/__builtin__/importing.py",
line 499, in load_compiled_module
code_w.exec_code(space, w_dic, w_dic)
File "/var/tmp/pypy-svn/dist/pypy/interpreter/eval.py", line 27, in
exec_code
return frame.run()
File "/var/tmp/pypy-svn/dist/pypy/interpreter/pyframe.py", line 91, in
run
return self.execute_frame()
File "/var/tmp/pypy-svn/dist/pypy/interpreter/pyframe.py", line 117,
in execute_frame
executioncontext)
File "/var/tmp/pypy-svn/dist/pypy/interpreter/pyopcode.py", line 79,
in dispatch
next_instr = self.handle_bytecode(co_code, next_instr, ec)
File "/var/tmp/pypy-svn/dist/pypy/interpreter/pyopcode.py", line 89,
in handle_bytecode
next_instr = self.dispatch_bytecode(co_code, next_instr, ec)
File "/var/tmp/pypy-svn/dist/pypy/interpreter/pyopcode.py", line 235,
in dispatch_bytecode
res = getattr(self, methodname)(oparg, next_instr)
File "/var/tmp/pypy-svn/dist/pypy/interpreter/pyopcode.py", line 769,
in IMPORT_STAR
import_all_from(f.space, w_module, w_locals)
File "/var/tmp/pypy-svn/dist/pypy/interpreter/gateway.py", line 839,
in appcaller
return space.call_args(w_func, args)
File "/var/tmp/pypy-svn/dist/pypy/objspace/descroperation.py", line
98, in call_args
return w_obj.call_args(args)
File "/var/tmp/pypy-svn/dist/pypy/interpreter/function.py", line 37,
in call_args
return self.code.funcrun(self, args)
File "/var/tmp/pypy-svn/dist/pypy/interpreter/gateway.py", line 483,
in funcrun
return BuiltinCode.funcrun_obj(self, func, None, args)
File "/var/tmp/pypy-svn/dist/pypy/interpreter/gateway.py", line 491,
in funcrun_obj
w_result = activation._run(space, scope_w)
File "</var/tmp/pypy-svn/dist/py/code/source.py:213>", line 3, in
_run_UWS_ObjSpace_W_Root_W_Root
File
"/var/tmp/pypy-svn/dist/pypy/_cache/pyopcode_4922ecf55c7db5d81d0c4a63f92bdd44.py",
line 59, in import_all_from
w_2 = space.getattr(w_module, gs___dict__)
File "/var/tmp/pypy-svn/dist/pypy/objspace/std/objspace.py", line 669,
in getattr
return DescrOperation.getattr(self, w_obj, w_name)
File "/var/tmp/pypy-svn/dist/pypy/objspace/descroperation.py", line
134, in getattr
return space._handle_getattribute(w_descr, w_obj, w_name)
File "/var/tmp/pypy-svn/dist/pypy/objspace/descroperation.py", line
140, in _handle_getattribute
return space.get_and_call_function(w_descr, w_obj, w_name)
File "/var/tmp/pypy-svn/dist/pypy/objspace/descroperation.py", line
89, in get_and_call_function
return descr.funccall(w_obj, *args_w)
File "/var/tmp/pypy-svn/dist/pypy/interpreter/function.py", line 69,
in funccall
return self.call_args(Arguments(self.space, list(args_w)))
File "/var/tmp/pypy-svn/dist/pypy/interpreter/function.py", line 37,
in call_args
return self.code.funcrun(self, args)
File "/var/tmp/pypy-svn/dist/pypy/interpreter/eval.py", line 61, in
funcrun
return frame.run()
File "/var/tmp/pypy-svn/dist/pypy/objspace/std/fake.py", line 171, in
run
return self.space.wrap(result)
File "/var/tmp/pypy-svn/dist/pypy/objspace/std/objspace.py", line 475,
in wrap
items_w = [(self.wrap(k), self.wrap(v)) for (k, v) in x.iteritems()]
File "/var/tmp/pypy-svn/dist/pypy/objspace/std/objspace.py", line 522,
in wrap
return fake_object(self, x)
File "/var/tmp/pypy-svn/dist/pypy/objspace/std/fake.py", line 20, in
fake_object
ft = fake_type(x)
File "/var/tmp/pypy-svn/dist/pypy/objspace/std/fake.py", line 58, in
fake_type
faked_type = really_build_fake_type(cpy_type)
File "/var/tmp/pypy-svn/dist/pypy/objspace/std/fake.py", line 108, in
really_build_fake_type
assert cpy_type.__base__ is basestring
AssertionError
_______________________________________________
[email protected]
http://codespeak.net/mailman/listinfo/pypy-dev
