Author: Edd Barrett <[email protected]>
Branch: refactor_rmmap
Changeset: r86169:a80e988edfd0
Date: 2016-08-12 11:26 +0100
http://bitbucket.org/pypy/pypy/changeset/a80e988edfd0/
Log: Refactor rmmap.py JIT support into its own file.
And test with a W^X patch to the build system.
diff --git a/rpython/jit/backend/llsupport/asmmemmgr.py
b/rpython/jit/backend/llsupport/asmmemmgr.py
--- a/rpython/jit/backend/llsupport/asmmemmgr.py
+++ b/rpython/jit/backend/llsupport/asmmemmgr.py
@@ -1,7 +1,7 @@
import sys
from rpython.rlib.rarithmetic import intmask, r_uint, LONG_BIT
from rpython.rlib.objectmodel import we_are_translated
-from rpython.rlib import rmmap
+from rpython.jit.backend.llsupport import rmmap
from rpython.rlib.debug import debug_start, debug_print, debug_stop
from rpython.rlib.debug import have_debug_prints
from rpython.rtyper.lltypesystem import lltype, rffi
diff --git a/rpython/jit/backend/llsupport/rmmap.py
b/rpython/jit/backend/llsupport/rmmap.py
new file mode 100644
--- /dev/null
+++ b/rpython/jit/backend/llsupport/rmmap.py
@@ -0,0 +1,145 @@
+"""mmap for the JIT
+
+Derived from rlib.rmmap
+"""
+
+# borrow a few bits from our rlib cousin, but we must not share functions
+from rpython.rlib.rmmap import _POSIX, _MS_WINDOWS, _CYGWIN, constants, CConfig
+if _POSIX:
+ from rpython.rlib.rmmap import (
+ MAP_PRIVATE, MAP_ANONYMOUS, PROT_EXEC, PROT_READ, PROT_WRITE, PTR)
+if _MS_WINDOWS:
+ from rpython.rlib.rwin32 import LPDWORD, DWORD, BOOL
+ from rpython.rlib.rmmap import (
+ MEM_COMMIT, MEM_RESERVE, PAGE_EXECUTE_READWRITE, MEM_RELEASE)
+
+from rpython.rtyper.lltypesystem import rffi, lltype
+from rpython.rlib import rposix
+from rpython.rtyper.tool import rffi_platform
+
+
+locals().update(constants)
+
+
+def safe_external(name, args, result, save_err_on_unsafe=0, save_err_on_safe=0,
+ **kwargs):
+ return rffi.llexternal(name, args, result,
+ compilation_info=CConfig._compilation_info_,
+ sandboxsafe=True, releasegil=False,
+ save_err=save_err_on_safe, **kwargs)
+
+
+def safe_winexternal(name, args, result, **kwargs):
+ return rffi.llexternal(name, args, result,
+ compilation_info=CConfig._compilation_info_,
+ calling_conv='win', sandboxsafe=True,
+ releasegil=False, **kwargs)
+
+
+if _POSIX:
+ c_mmap_safe = safe_external(
+ 'mmap', [PTR, size_t, rffi.INT, rffi.INT, rffi.INT, off_t], PTR,
+ macro=True, save_err_on_unsafe=rffi.RFFI_SAVE_ERRNO)
+
+ c_munmap_safe = safe_external('munmap', [PTR, size_t], rffi.INT)
+
+
+if _CYGWIN:
+ c_free_safe = safe_external('free', [PTR], lltype.Void, macro=True)
+ c_malloc_safe = safe_external('malloc', [size_t], PTR, macro=True)
+
+if _MS_WINDOWS:
+ VirtualAlloc_safe = safe_winexternal(
+ 'VirtualAlloc', [rffi.VOIDP, rffi.SIZE_T, DWORD, DWORD], rffi.VOIDP)
+
+ _VirtualProtect_safe = safe_winexternal(
+ 'VirtualProtect', [rffi.VOIDP, rffi.SIZE_T, DWORD, LPDWORD], BOOL)
+
+ def VirtualProtect(addr, size, mode, oldmode_ptr):
+ return _VirtualProtect_safe(
+ addr, rffi.cast(rffi.SIZE_T, size), rffi.cast(DWORD, mode),
+ oldmode_ptr)
+ VirtualProtect._annspecialcase_ = 'specialize:ll'
+
+ VirtualFree_safe = safe_winexternal(
+ 'VirtualFree', [rffi.VOIDP, rffi.SIZE_T, DWORD], BOOL)
+
+
+if _POSIX:
+ def alloc_hinted(hintp, map_size):
+ flags = MAP_PRIVATE | MAP_ANONYMOUS
+ prot = PROT_EXEC | PROT_READ | PROT_WRITE
+ return c_mmap_safe(hintp, map_size, prot, flags, -1, 0)
+
+ # XXX is this really necessary?
+ class Hint:
+ pos = -0x4fff0000 # for reproducible results
+ hint = Hint()
+
+ def alloc(map_size):
+ """Allocate memory. This is intended to be used by the JIT,
+ so the memory has the executable bit set and gets allocated
+ internally in case of a sandboxed process.
+ """
+ from errno import ENOMEM
+ from rpython.rlib import debug
+
+ if _CYGWIN:
+ # XXX: JIT memory should be using mmap MAP_PRIVATE with
+ # PROT_EXEC but Cygwin's fork() fails. mprotect()
+ # cannot be used, but seems to be unnecessary there.
+ res = c_malloc_safe(map_size)
+ if res == rffi.cast(PTR, 0):
+ raise MemoryError
+ return res
+ res = alloc_hinted(rffi.cast(PTR, hint.pos), map_size)
+ if res == rffi.cast(PTR, -1):
+ # some systems (some versions of OS/X?) complain if they
+ # are passed a non-zero address. Try again.
+ res = alloc_hinted(rffi.cast(PTR, 0), map_size)
+ if res == rffi.cast(PTR, -1):
+ # ENOMEM simply raises MemoryError, but other errors are fatal
+ if rposix.get_saved_errno() != ENOMEM:
+ debug.fatalerror_notb(
+ "Got an unexpected error trying to allocate some "
+ "memory for the JIT (tried to do mmap() with "
+ "PROT_EXEC|PROT_READ|PROT_WRITE). This can be caused "
+ "by a system policy like PAX. You need to find how "
+ "to work around the policy on your system.")
+ raise MemoryError
+ else:
+ hint.pos += map_size
+ return res
+ alloc._annenforceargs_ = (int,)
+
+ if _CYGWIN:
+ free = c_free_safe
+ else:
+ free = c_munmap_safe
+
+elif _MS_WINDOWS:
+ class Hint:
+ pos = -0x4fff0000 # for reproducible results
+ hint = Hint()
+ # XXX this has no effect on windows
+
+ def alloc(map_size):
+ """Allocate memory. This is intended to be used by the JIT,
+ so the memory has the executable bit set.
+ XXX implement me: it should get allocated internally in
+ case of a sandboxed process
+ """
+ null = lltype.nullptr(rffi.VOIDP.TO)
+ res = VirtualAlloc_safe(null, map_size, MEM_COMMIT | MEM_RESERVE,
+ PAGE_EXECUTE_READWRITE)
+ if not res:
+ raise MemoryError
+ arg = lltype.malloc(LPDWORD.TO, 1, zero=True, flavor='raw')
+ VirtualProtect(res, map_size, PAGE_EXECUTE_READWRITE, arg)
+ lltype.free(arg, flavor='raw')
+ # ignore errors, just try
+ return res
+ alloc._annenforceargs_ = (int,)
+
+ def free(ptr, map_size):
+ VirtualFree_safe(ptr, 0, MEM_RELEASE)
diff --git a/rpython/jit/backend/x86/detect_feature.py
b/rpython/jit/backend/x86/detect_feature.py
--- a/rpython/jit/backend/x86/detect_feature.py
+++ b/rpython/jit/backend/x86/detect_feature.py
@@ -1,17 +1,20 @@
import sys
import struct
from rpython.rtyper.lltypesystem import lltype, rffi
-from rpython.rlib.rmmap import alloc, free
+from rpython.rlib.rmmap import alloc, free, set_pages_executable
+
+CPU_INFO_SZ = 4096
def cpu_info(instr):
- data = alloc(4096)
+ data = alloc(CPU_INFO_SZ, no_exec=True)
pos = 0
for c in instr:
data[pos] = c
pos += 1
+ set_pages_executable(data, CPU_INFO_SZ)
fnptr = rffi.cast(lltype.Ptr(lltype.FuncType([], lltype.Signed)), data)
code = fnptr()
- free(data, 4096)
+ free(data, CPU_INFO_SZ)
return code
def detect_sse2():
diff --git a/rpython/rlib/rmmap.py b/rpython/rlib/rmmap.py
--- a/rpython/rlib/rmmap.py
+++ b/rpython/rlib/rmmap.py
@@ -155,6 +155,8 @@
c_mmap, c_mmap_safe = external('mmap', [PTR, size_t, rffi.INT, rffi.INT,
rffi.INT, off_t], PTR, macro=True,
save_err_on_unsafe=rffi.RFFI_SAVE_ERRNO)
+ c_mprotect, _ = external('mprotect',
+ [PTR, size_t, rffi.INT], rffi.INT)
# 'mmap' on linux32 is a macro that calls 'mmap64'
_, c_munmap_safe = external('munmap', [PTR, size_t], rffi.INT)
c_msync, _ = external('msync', [PTR, size_t, rffi.INT], rffi.INT,
@@ -705,14 +707,21 @@
m.setdata(res, map_size)
return m
- def alloc_hinted(hintp, map_size):
+ def alloc_hinted(hintp, map_size, no_exec=False):
flags = MAP_PRIVATE | MAP_ANONYMOUS
- prot = PROT_EXEC | PROT_READ | PROT_WRITE
+ prot = PROT_READ | PROT_WRITE
+ if not no_exec:
+ prot |= PROT_EXEC
if we_are_translated():
flags = NonConstant(flags)
prot = NonConstant(prot)
return c_mmap_safe(hintp, map_size, prot, flags, -1, 0)
+ def set_pages_executable(addr, size):
+ rv = c_mprotect(addr, size, PROT_EXEC)
+ if rv < 0:
+ debug.fatalerror_notb("set_pages_executable failed")
+
def clear_large_memory_chunk_aligned(addr, map_size):
addr = rffi.cast(PTR, addr)
flags = MAP_FIXED | MAP_PRIVATE | MAP_ANONYMOUS
@@ -728,7 +737,7 @@
pos = -0x4fff0000 # for reproducible results
hint = Hint()
- def alloc(map_size):
+ def alloc(map_size, no_exec=False):
"""Allocate memory. This is intended to be used by the JIT,
so the memory has the executable bit set and gets allocated
internally in case of a sandboxed process.
@@ -744,11 +753,11 @@
if res == rffi.cast(PTR, 0):
raise MemoryError
return res
- res = alloc_hinted(rffi.cast(PTR, hint.pos), map_size)
+ res = alloc_hinted(rffi.cast(PTR, hint.pos), map_size, no_exec=no_exec)
if res == rffi.cast(PTR, -1):
# some systems (some versions of OS/X?) complain if they
# are passed a non-zero address. Try again.
- res = alloc_hinted(rffi.cast(PTR, 0), map_size)
+ res = alloc_hinted(rffi.cast(PTR, 0), map_size, no_exec=no_exec)
if res == rffi.cast(PTR, -1):
# ENOMEM simply raises MemoryError, but other errors are fatal
if rposix.get_saved_errno() != ENOMEM:
@@ -762,7 +771,7 @@
else:
hint.pos += map_size
return res
- alloc._annenforceargs_ = (int,)
+ alloc._annenforceargs_ = (int, bool)
if _CYGWIN:
free = c_free_safe
@@ -933,11 +942,13 @@
hint = Hint()
# XXX this has no effect on windows
- def alloc(map_size):
+ def alloc(map_size, no_exec=no_exec):
"""Allocate memory. This is intended to be used by the JIT,
so the memory has the executable bit set.
XXX implement me: it should get allocated internally in
case of a sandboxed process
+
+ XXX no_exec not implemented on windows
"""
null = lltype.nullptr(rffi.VOIDP.TO)
res = VirtualAlloc_safe(null, map_size, MEM_COMMIT | MEM_RESERVE,
@@ -949,7 +960,10 @@
lltype.free(arg, flavor='raw')
# ignore errors, just try
return res
- alloc._annenforceargs_ = (int,)
+ alloc._annenforceargs_ = (int, bool)
+
+ def set_pages_executable(addr, size):
+ pass # XXX not implemented on windows
def free(ptr, map_size):
VirtualFree_safe(ptr, 0, MEM_RELEASE)
_______________________________________________
pypy-commit mailing list
[email protected]
https://mail.python.org/mailman/listinfo/pypy-commit
