Author: Armin Rigo <ar...@tunes.org> Branch: stacklet Changeset: r46721:a2a7b3bc7f91 Date: 2011-08-23 00:18 +0200 http://bitbucket.org/pypy/pypy/changeset/a2a7b3bc7f91/
Log: Shadowstack support for stacklets. Phew. Only missing: see "not implemented yet" in shadowstack.py. diff --git a/pypy/rlib/_stacklet_shadowstack.py b/pypy/rlib/_stacklet_shadowstack.py --- a/pypy/rlib/_stacklet_shadowstack.py +++ b/pypy/rlib/_stacklet_shadowstack.py @@ -1,256 +1,114 @@ from pypy.rlib import _rffi_stacklet as _c -from pypy.rpython.lltypesystem import lltype, llmemory, rffi +from pypy.rlib.debug import ll_assert +from pypy.rpython.annlowlevel import llhelper +from pypy.rpython.lltypesystem import lltype, llmemory from pypy.rpython.lltypesystem.lloperation import llop -from pypy.rlib import rstacklet, rgc -from pypy.rlib.debug import ll_assert, debug_print -from pypy.rpython.annlowlevel import llhelper +from pypy.tool.staticmethods import StaticMethods -DEBUG = False - -PTR_SUSPSTACK = lltype.Ptr(lltype.GcForwardReference()) SUSPSTACK = lltype.GcStruct('SuspStack', ('handle', _c.handle), - ('shadowstack_stop', llmemory.Address), # low (old) - ('shadowstack_start', llmemory.Address),# high(new) - ('shadowstack_saved', lltype.Signed), # number - ('shadowstack_prev', PTR_SUSPSTACK), - # copy of the shadowstack, but in reversed order: - # the first items here are the last items in the - # real shadowstack. Only items 0 to saved-1 are - # set up; the rest is still in the real shadowstack - ('shadowstack_copy', lltype.Array(llmemory.GCREF))) -PTR_SUSPSTACK.TO.become(SUSPSTACK) + ('shadowstackref', llmemory.GCREF)) NULL_SUSPSTACK = lltype.nullptr(SUSPSTACK) +NULL_GCREF = lltype.nullptr(llmemory.GCREF.TO) -sizeofaddr = llmemory.sizeof(llmemory.Address) -def repr_suspstack(suspstack): - debug_print('<SuspStack', - rffi.cast(lltype.Signed, suspstack.handle), - ': stop', - rffi.cast(lltype.Signed, suspstack.shadowstack_stop), - 'start', - rffi.cast(lltype.Signed, suspstack.shadowstack_start), - 'saved', - suspstack.shadowstack_saved, - '>') +def _new_callback(h, arg): + # We still have the old shadowstack active at this point; save it + # away, and start a fresh new one + oldsuspstack = gcrootfinder.oldsuspstack + oldsuspstack.handle = h + llop.gc_save_current_state_away(lltype.Void, + oldsuspstack.shadowstackref) + llop.gc_start_fresh_new_state(lltype.Void) + gcrootfinder.oldsuspstack = NULL_SUSPSTACK + # + newsuspstack = gcrootfinder.callback(oldsuspstack, arg) + # + # Finishing this stacklet. + gcrootfinder.oldsuspstack = NULL_SUSPSTACK + gcrootfinder.newsuspstack = newsuspstack + return newsuspstack.handle -# every thread has a 'current stack stop', which is like -# g_current_stack_stop in stacklet.c but about the shadowstack -rstacklet.StackletThread._current_shadowstack_stop = llmemory.NULL +def prepare_old_suspstack(): + if not gcrootfinder.oldsuspstack: # else reuse the one still there + _allocate_old_suspstack() -# every thread has a 'stack chain' too, similar to g_stack_chain_head. -rstacklet.StackletThread._shadowstack_chain_head = NULL_SUSPSTACK +def _allocate_old_suspstack(): + suspstack = lltype.malloc(SUSPSTACK) + suspstack.shadowstackref = llop.gc_new_shadowstackref(llmemory.GCREF) + gcrootfinder.oldsuspstack = suspstack +_allocate_old_suspstack._dont_inline_ = True +def get_result_suspstack(h): + # Now we are in the target, after the switch() or the new(). + # Note that this whole module was carefully written in such a way as + # not to invoke pushing/popping things off the shadowstack at + # unexpected moments... + oldsuspstack = gcrootfinder.oldsuspstack + newsuspstack = gcrootfinder.newsuspstack + gcrootfinder.oldsuspstack = NULL_SUSPSTACK + gcrootfinder.newsuspstack = NULL_SUSPSTACK + if not h: + raise MemoryError + # We still have the old shadowstack active at this point; save it + # away, and restore the new one + if oldsuspstack: + ll_assert(not _c.is_empty_handle(h),"unexpected empty stacklet handle") + oldsuspstack.handle = h + llop.gc_save_current_state_away(lltype.Void, + oldsuspstack.shadowstackref) + else: + ll_assert(_c.is_empty_handle(h),"unexpected non-empty stacklet handle") + llop.gc_forget_current_state(lltype.Void) + # + llop.gc_restore_state_from(lltype.Void, newsuspstack.shadowstackref) + # + # From this point on, 'newsuspstack' is consumed and done, its + # shadow stack installed as the current one. It should not be + # used any more. For performance, we avoid it being deallocated + # by letting it be reused on the next switch. + gcrootfinder.oldsuspstack = newsuspstack + # Return. + return oldsuspstack -def root_stack_top(): - return llop.gc_adr_of_root_stack_top(llmemory.Address).address[0] -def set_root_stack_top(newaddr): - llop.gc_adr_of_root_stack_top(llmemory.Address).address[0] = newaddr +class StackletGcRootFinder: + __metaclass__ = StaticMethods + def new(thrd, callback, arg): + gcrootfinder.callback = callback + thread_handle = thrd._thrd + prepare_old_suspstack() + h = _c.new(thread_handle, llhelper(_c.run_fn, _new_callback), arg) + return get_result_suspstack(h) + new._dont_inline_ = True -def _new_runfn(h, arg): - thrd = gcrootfinder.thrd - thrd._current_shadowstack_stop = root_stack_top() - suspstack = gcrootfinder.attach_handle_on_suspstack(h) - # - suspstack = gcrootfinder.runfn(suspstack, arg) - # - gcrootfinder.thrd = thrd - gcrootfinder.g_source = NULL_SUSPSTACK - gcrootfinder.g_target = suspstack - return gcrootfinder.consume_suspstack() + def switch(thrd, suspstack): + # suspstack has a handle to target, i.e. where to switch to + ll_assert(suspstack != gcrootfinder.oldsuspstack, + "stacklet: invalid use") + gcrootfinder.newsuspstack = suspstack + thread_handle = thrd._thrd + h = suspstack.handle + prepare_old_suspstack() + h = _c.switch(thread_handle, h) + return get_result_suspstack(h) + switch._dont_inline_ = True - -class StackletGcRootFinder(object): - g_source = NULL_SUSPSTACK - g_target = NULL_SUSPSTACK - - def new(self, thrd, callback, arg): - rst = root_stack_top() - if (thrd._current_shadowstack_stop == llmemory.NULL or - thrd._current_shadowstack_stop > rst): - thrd._current_shadowstack_stop = rst - self.allocate_source_suspstack(rst, thrd) - self.runfn = callback - h = _c.new(thrd._thrd, llhelper(_c.run_fn, _new_runfn), arg) - return self.get_result_suspstack(h, False) - - def switch(self, thrd, suspstack): - rst = root_stack_top() - if thrd._current_shadowstack_stop > rst: - thrd._current_shadowstack_stop = rst - self.allocate_source_suspstack(rst, thrd) - self.g_target = suspstack - h = self.consume_suspstack() - h2 = _c.switch(thrd._thrd, h) - return self.get_result_suspstack(h2, True) - - def allocate_source_suspstack(self, rst, thrd): - # Attach to 'self' a SUSPSTACK that represents the - # old, but still current, stacklet. All that is left to - # fill is 'self.g_source.handle', done later by a call - # to attach_handle_on_suspstack(). - count = (rst - thrd._current_shadowstack_stop) // sizeofaddr - newsuspstack = lltype.malloc(SUSPSTACK, count) - newsuspstack.shadowstack_stop = thrd._current_shadowstack_stop - newsuspstack.shadowstack_start = rst - newsuspstack.shadowstack_saved = 0 - newsuspstack.shadowstack_prev = thrd._shadowstack_chain_head - # - newsuspstack.handle = _c.null_handle - if DEBUG: - debug_print("NEW") - repr_suspstack(newsuspstack) - # - thrd._shadowstack_chain_head = newsuspstack - self.g_source = newsuspstack - self.thrd = thrd - allocate_source_suspstack._dont_inline_ = True - # ^^^ dont_inline because it has a malloc, so we want its effects on - # the root_stack_top to be isolated - - @rgc.no_collect - def attach_handle_on_suspstack(self, handle): - s = self.g_source - self.g_source = NULL_SUSPSTACK - s.handle = handle - if DEBUG: - debug_print('ATTACH HANDLE') - repr_suspstack(s) - return s - - @rgc.no_collect - def get_result_suspstack(self, h, restore_if_out_of_memory): - # - # Return from a new() or a switch(): 'h' is a handle, possibly - # an empty one, that says from where we switched to. - if not h: - # oups, we didn't actually switch anywhere, but just got - # an out-of-memory condition. Restore the current suspstack. - if restore_if_out_of_memory: - self.g_target = self.g_source - self.consume_suspstack() - self.g_source = NULL_SUSPSTACK - self.g_target = NULL_SUSPSTACK - self.thrd = None - raise MemoryError - # - ll_assert(self.g_target == NULL_SUSPSTACK, "g_target must be cleared") - # - if _c.is_empty_handle(h): - ll_assert(self.g_source == NULL_SUSPSTACK, - "g_source must be cleared") - self.thrd = None - return NULL_SUSPSTACK - else: - # This is a return that gave us a real handle. Store it. - return self.attach_handle_on_suspstack(h) - - @rgc.no_collect - def consume_suspstack(self): - if DEBUG: - debug_print('CONSUME') - repr_suspstack(self.g_target) - # - # We want to switch to or return to 'suspstack'. First get - # how far we have to clean up the shadowstack. - target = self.possibly_move_back() - # - # Clean the shadowstack up to that position. - self.clear_shadowstack(target) - # - # Now restore data from suspstack.shadowstack_copy. - self.restore_suspstack() - # - # Set the new root stack bounds. - suspstack = self.g_target - self.thrd._current_shadowstack_stop = target - set_root_stack_top(suspstack.shadowstack_start) - # - # Now the real shadowstack is ready for 'suspstack'. - self.g_target = NULL_SUSPSTACK - return suspstack.handle - - def clear_shadowstack(self, target_stop): - # NB. see also g_clear_stack() in stacklet.c. - # - current = self.thrd._shadowstack_chain_head - # - # save and unlink suspstacks that are completely within - # the area to clear. - while bool(current) and current.shadowstack_stop >= target_stop: - prev = current.shadowstack_prev - current.shadowstack_prev = NULL_SUSPSTACK - if current != self.g_target: - # don't bother saving away targetsuspstack, because - # it would be immediately restored - self._save(current, current.shadowstack_stop) - current = prev - # - # save a partial stack - if bool(current) and current.shadowstack_start > target_stop: - self._save(current, target_stop) - # - self.thrd._shadowstack_chain_head = current - - def _save(self, suspstack, stop): - # See g_save() in stacklet.c. - num1 = suspstack.shadowstack_saved - num2 = (suspstack.shadowstack_start - stop) // sizeofaddr - ll_assert(stop >= suspstack.shadowstack_stop, "stacklet+shadowstack#1") - source = suspstack.shadowstack_start - num1 * sizeofaddr - while num1 < num2: - source -= sizeofaddr - addr = source.address[0] - gcref = llmemory.cast_adr_to_ptr(addr, llmemory.GCREF) - suspstack.shadowstack_copy[num1] = gcref - num1 += 1 - suspstack.shadowstack_saved = num1 - - def restore_suspstack(self): - suspstack = self.g_target - target = suspstack.shadowstack_start - saved = suspstack.shadowstack_saved - suspstack.shadowstack_saved = 0 - i = 0 - while i < saved: - addr = llmemory.cast_ptr_to_adr(suspstack.shadowstack_copy[i]) - target -= sizeofaddr - target.address[0] = addr - i += 1 - - def possibly_move_back(self): - # if suspstack.shadowstack_stop is after root_stack_top, then - # restoring it there would leave an uninitialized gap containing - # garbage in the real shadowstack. Avoid this by shifting - # susp.shadowstack_{stop,start}. - suspstack = self.g_target - rst = root_stack_top() - delta = suspstack.shadowstack_stop - rst - if delta > 0: - # completely saved in this case - ll_assert(suspstack.shadowstack_start - suspstack.shadowstack_stop - == suspstack.shadowstack_saved * sizeofaddr, - "not completely saved shadowstack?") - if DEBUG: - debug_print('moving to the left by', delta) - suspstack.shadowstack_stop -= delta - suspstack.shadowstack_start -= delta - return suspstack.shadowstack_stop - - def destroy(self, thrd, suspstack): + def destroy(thrd, suspstack): h = suspstack.handle suspstack.handle = _c.null_handle + suspstack.shadowstackref = NULL_GCREF _c.destroy(thrd._thrd, h) - def is_empty_handle(self, suspstack): + def is_empty_handle(suspstack): return not suspstack - def get_null_handle(self): + def get_null_handle(): return NULL_SUSPSTACK gcrootfinder = StackletGcRootFinder() +gcrootfinder.oldsuspstack = NULL_SUSPSTACK +gcrootfinder.newsuspstack = NULL_SUSPSTACK diff --git a/pypy/rpython/llinterp.py b/pypy/rpython/llinterp.py --- a/pypy/rpython/llinterp.py +++ b/pypy/rpython/llinterp.py @@ -825,10 +825,11 @@ def op_gc_adr_of_nursery_top(self): raise NotImplementedError - def op_gc_adr_of_nursery_free(self): raise NotImplementedError + def op_gc_adr_of_root_stack_base(self): + raise NotImplementedError def op_gc_adr_of_root_stack_top(self): raise NotImplementedError @@ -879,6 +880,17 @@ def op_gc_stack_bottom(self): pass # marker for trackgcroot.py + def op_gc_new_shadowstackref(self): # stacklet+shadowstack + raise NotImplementedError("gc_new_shadowstackref") + def op_gc_save_current_state_away(self): + raise NotImplementedError("gc_save_current_state_away") + def op_gc_forget_current_state(self): + raise NotImplementedError("gc_forget_current_state") + def op_gc_restore_state_from(self): + raise NotImplementedError("gc_restore_state_from") + def op_gc_start_fresh_new_state(self): + raise NotImplementedError("gc_start_fresh_new_state") + def op_gc_get_type_info_group(self): raise NotImplementedError("gc_get_type_info_group") diff --git a/pypy/rpython/lltypesystem/lloperation.py b/pypy/rpython/lltypesystem/lloperation.py --- a/pypy/rpython/lltypesystem/lloperation.py +++ b/pypy/rpython/lltypesystem/lloperation.py @@ -488,6 +488,13 @@ 'gc_asmgcroot_static': LLOp(sideeffects=False), 'gc_stack_bottom': LLOp(canrun=True), + # for stacklet+shadowstack support + 'gc_new_shadowstackref': LLOp(canmallocgc=True), + 'gc_save_current_state_away': LLOp(), + 'gc_forget_current_state': LLOp(), + 'gc_restore_state_from': LLOp(), + 'gc_start_fresh_new_state': LLOp(), + # NOTE NOTE NOTE! don't forget *** canmallocgc=True *** for anything that # can malloc a GC object. diff --git a/pypy/rpython/memory/gctransform/asmgcroot.py b/pypy/rpython/memory/gctransform/asmgcroot.py --- a/pypy/rpython/memory/gctransform/asmgcroot.py +++ b/pypy/rpython/memory/gctransform/asmgcroot.py @@ -147,7 +147,7 @@ self._extra_gcmapend = lambda: llmemory.NULL self._extra_mark_sorted = lambda: True - def need_stacklet_support(self): + def need_stacklet_support(self, gctransformer, getfn): # stacklet support: BIG HACK for rlib.rstacklet from pypy.rlib import _stacklet_asmgcc _stacklet_asmgcc._asmstackrootwalker = self # as a global! argh diff --git a/pypy/rpython/memory/gctransform/framework.py b/pypy/rpython/memory/gctransform/framework.py --- a/pypy/rpython/memory/gctransform/framework.py +++ b/pypy/rpython/memory/gctransform/framework.py @@ -479,7 +479,7 @@ # thread support if translator.config.translation.continuation: - root_walker.need_stacklet_support() + root_walker.need_stacklet_support(self, getfn) if translator.config.translation.thread: root_walker.need_thread_support(self, getfn) @@ -797,6 +797,33 @@ def gct_gc_adr_of_root_stack_top(self, hop): self._gc_adr_of_gcdata_attr(hop, 'root_stack_top') + def gct_gc_new_shadowstackref(self, hop): + op = hop.spaceop + livevars = self.push_roots(hop) + hop.genop("direct_call", [self.root_walker.gc_new_shadowstackref_ptr], + resultvar=op.result) + self.pop_roots(hop, livevars) + + def gct_gc_save_current_state_away(self, hop): + op = hop.spaceop + hop.genop("direct_call", + [self.root_walker.gc_save_current_state_away_ptr, + op.args[0]]) + + def gct_gc_forget_current_state(self, hop): + hop.genop("direct_call", + [self.root_walker.gc_forget_current_state_ptr]) + + def gct_gc_restore_state_from(self, hop): + op = hop.spaceop + hop.genop("direct_call", + [self.root_walker.gc_restore_state_from_ptr, + op.args[0]]) + + def gct_gc_start_fresh_new_state(self, hop): + hop.genop("direct_call", + [self.root_walker.gc_start_fresh_new_state_ptr]) + def gct_gc_x_swap_pool(self, hop): raise NotImplementedError("old operation deprecated") def gct_gc_x_clone(self, hop): diff --git a/pypy/rpython/memory/gctransform/shadowstack.py b/pypy/rpython/memory/gctransform/shadowstack.py --- a/pypy/rpython/memory/gctransform/shadowstack.py +++ b/pypy/rpython/memory/gctransform/shadowstack.py @@ -2,7 +2,7 @@ from pypy.rpython.memory.gctransform.framework import sizeofaddr from pypy.rpython.annlowlevel import llhelper from pypy.rpython.lltypesystem import lltype, llmemory -from pypy.rpython.lltypesystem.lloperation import llop +from pypy.rlib.debug import ll_assert from pypy.annotation import model as annmodel @@ -65,12 +65,6 @@ self.rootstackhook(collect_stack_root, gcdata.root_stack_base, gcdata.root_stack_top) - def need_stacklet_support(self): - XXXXXX # FIXME - # stacklet support: BIG HACK for rlib.rstacklet - from pypy.rlib import _stacklet_shadowstack - _stacklet_shadowstack._shadowstackrootwalker = self # as a global! argh - def need_thread_support(self, gctransformer, getfn): from pypy.module.thread import ll_thread # xxx fish from pypy.rpython.memory.support import AddressDict @@ -81,7 +75,7 @@ # gc_thread_run and gc_thread_die. See docstrings below. shadow_stack_pool = self.shadow_stack_pool - SHADOWSTACKREF = make_shadowstackref(gctransformer) + SHADOWSTACKREF = get_shadowstackref(gctransformer) # this is a dict {tid: SHADOWSTACKREF}, where the tid for the # current thread may be missing so far @@ -182,7 +176,41 @@ annmodel.SomeAddress()], annmodel.s_None, minimal_transform=False) - self.has_thread_support = True + + def need_stacklet_support(self, gctransformer, getfn): + shadow_stack_pool = self.shadow_stack_pool + SHADOWSTACKREF = get_shadowstackref(gctransformer) + + def gc_new_shadowstackref(): + ssref = shadow_stack_pool.allocate(SHADOWSTACKREF) + return lltype.cast_opaque_ptr(llmemory.GCREF, ssref) + + def gc_save_current_state_away(gcref): + ssref = lltype.cast_opaque_ptr(lltype.Ptr(SHADOWSTACKREF), gcref) + shadow_stack_pool.save_current_state_away(ssref) + + def gc_forget_current_state(): + shadow_stack_pool.forget_current_state() + + def gc_restore_state_from(gcref): + ssref = lltype.cast_opaque_ptr(lltype.Ptr(SHADOWSTACKREF), gcref) + shadow_stack_pool.restore_state_from(ssref) + + def gc_start_fresh_new_state(): + shadow_stack_pool.start_fresh_new_state() + + s_gcref = annmodel.SomePtr(llmemory.GCREF) + self.gc_new_shadowstackref_ptr = getfn(gc_new_shadowstackref, + [], s_gcref, + minimal_transform=False) + self.gc_save_current_state_away_ptr = getfn(gc_save_current_state_away, + [s_gcref], annmodel.s_None) + self.gc_forget_current_state_ptr = getfn(gc_forget_current_state, + [], annmodel.s_None) + self.gc_restore_state_from_ptr = getfn(gc_restore_state_from, + [s_gcref], annmodel.s_None) + self.gc_start_fresh_new_state_ptr = getfn(gc_start_fresh_new_state, + [], annmodel.s_None) # ____________________________________________________________ @@ -221,8 +249,17 @@ self._prepare_unused_stack() shadowstackref.base = self.gcdata.root_stack_base shadowstackref.top = self.gcdata.root_stack_top + ll_assert(shadowstackref.base <= shadowstackref.top, + "save_current_state_away: broken shadowstack") #shadowstackref.fullstack = True - llop.gc_assume_young_pointers(lltype.Void, shadowstackref) + # + # cannot use llop.gc_assume_young_pointers() here, because + # we are in a minimally-transformed GC helper :-/ + gc = self.gcdata.gc + if hasattr(gc.__class__, 'assume_young_pointers'): + shadowstackadr = llmemory.cast_ptr_to_adr(shadowstackref) + gc.assume_young_pointers(shadowstackadr) + # self.gcdata.root_stack_top = llmemory.NULL # to detect missing restore def forget_current_state(self): @@ -232,6 +269,9 @@ self.gcdata.root_stack_top = llmemory.NULL # to detect missing restore def restore_state_from(self, shadowstackref): + ll_assert(bool(shadowstackref.base), "empty shadowstackref!") + ll_assert(shadowstackref.base <= shadowstackref.top, + "restore_state_from: broken shadowstack") self.gcdata.root_stack_base = shadowstackref.base self.gcdata.root_stack_top = shadowstackref.top shadowstackref.base = llmemory.NULL @@ -249,7 +289,10 @@ raise MemoryError -def make_shadowstackref(gctransformer): +def get_shadowstackref(gctransformer): + if hasattr(gctransformer, '_SHADOWSTACKREF'): + return gctransformer._SHADOWSTACKREF + SHADOWSTACKREFPTR = lltype.Ptr(lltype.GcForwardReference()) SHADOWSTACKREF = lltype.GcStruct('ShadowStackRef', ('base', llmemory.Address), @@ -282,4 +325,5 @@ customtraceptr = llhelper(lltype.Ptr(CUSTOMTRACEFUNC), customtrace) lltype.attachRuntimeTypeInfo(SHADOWSTACKREF, customtraceptr=customtraceptr) + gctransformer._SHADOWSTACKREF = SHADOWSTACKREF return SHADOWSTACKREF diff --git a/pypy/translator/c/gc.py b/pypy/translator/c/gc.py --- a/pypy/translator/c/gc.py +++ b/pypy/translator/c/gc.py @@ -392,6 +392,9 @@ fieldname, funcgen.expr(c_skipoffset))) + def OP_GC_ASSUME_YOUNG_POINTERS(self, funcgen, op): + raise Exception("the FramewokGCTransformer should handle this") + class AsmGcRootFrameworkGcPolicy(FrameworkGcPolicy): transformerclass = asmgcroot.AsmGcRootFrameworkGCTransformer _______________________________________________ pypy-commit mailing list pypy-commit@python.org http://mail.python.org/mailman/listinfo/pypy-commit