Index: compiler/ncgset.pas =================================================================== --- compiler/ncgset.pas (revision 40548) +++ compiler/ncgset.pas (working copy) @@ -73,7 +73,14 @@ jumptable_no_range : boolean; { has the implementation jumptable support } min_label : tconstexprint; + { Number of labels } + labelcnt: TCgInt; + { Number of individual values checked, counting each value in a range + individually (e.g. 0..2 counts as 3). } + TrueCount: TCgInt; + function GetBranchLabel(Block: TNode; out _Label: TAsmLabel): Boolean; + function blocklabel(id:longint):tasmlabel; procedure optimizevalues(var max_linear_list:aint;var max_dist:aword);virtual; function has_jumptable : boolean;virtual; @@ -90,9 +97,9 @@ uses verbose, - symconst,symdef,defutil, + symconst,symdef,symsym,defutil, pass_2,tgobj, - ncon, + nbas,ncon,ncgflw, ncgutil,hlcgobj; @@ -524,6 +531,79 @@ TCGCASENODE *****************************************************************************} + + { Analyse the nodes following the else label - if empty, change to end label } + function tcgcasenode.GetBranchLabel(Block: TNode; out _Label: TAsmLabel): Boolean; + var + LabelSym: TLabelSym; + begin + Result := True; + + if not Assigned(Block) then + begin + { Block doesn't exist / is empty } + _Label := endlabel; + Exit; + end; + + { These optimisations aren't particularly debugger friendly } + if not (cs_opt_level2 in current_settings.optimizerswitches) then + begin + Result := False; + current_asmdata.getjumplabel(_Label); + Exit; + end; + + while Assigned(Block) do + begin + case Block.nodetype of + nothingn: + begin + _Label := endlabel; + Exit; + end; + goton: + begin + LabelSym := TCGGotoNode(Block).labelsym; + if not Assigned(LabelSym) then + InternalError(2018121131); + + _Label := TCGLabelNode(TCGGotoNode(Block).labelnode).getasmlabel; + if Assigned(_Label) then + { Keep tabs on the fact that an actual 'goto' was used } + Include(flowcontrol,fc_gotolabel) + else + Break; + Exit; + end; + blockn: + begin + Block := TBlockNode(Block).Left; + Continue; + end; + statementn: + begin + { If the right node is assigned, then it's a compound block + that can't be simplified, so fall through, set Result to + False and make a new label } + + if Assigned(TStatementNode(Block).right) then + Break; + + Block := TStatementNode(Block).Left; + Continue; + end; + end; + + Break; + end; + + { Create unique label } + Result := False; + current_asmdata.getjumplabel(_Label); + end; + + function tcgcasenode.blocklabel(id:longint):tasmlabel; begin if not assigned(blocks[id]) then @@ -642,7 +722,7 @@ opsize:=newdef; end; last:=0; - first:=true; + first:=(labelcnt > 1); { Can greatly simplify the range checks if there's only one label } scratch_reg:=hlcg.getintregister(current_asmdata.CurrAsmList,opsize); genitem(hp); hlcg.a_jmp_always(current_asmdata.CurrAsmList,elselabel); @@ -1043,15 +1123,28 @@ end; procedure tcgcasenode.pass_generate_code; + + { Combines "case_count_labels" and "case_true_count" } + procedure CountBoth(p : pcaselabel); + begin + Inc(labelcnt); + Inc(TrueCount, (p^._high.svalue - p^._low.svalue) + 1); + if assigned(p^.less) then + CountBoth(p^.less); + if assigned(p^.greater) then + CountBoth(p^.greater); + end; + var oldflowcontrol: tflowcontrol; i : longint; - dist,distv, + dist : aword; + distv, lv,hv, max_label: tconstexprint; - labelcnt : tcgint; max_linear_list : aint; max_dist : aword; + ShortcutElse: Boolean; begin location_reset(location,LOC_VOID,OS_NO); @@ -1058,10 +1151,15 @@ oldflowcontrol := flowcontrol; include(flowcontrol,fc_inflowcontrol); { Allocate labels } + current_asmdata.getjumplabel(endlabel); - current_asmdata.getjumplabel(elselabel); + + { Do some optimisation to deal with empty else blocks } + ShortcutElse := GetBranchLabel(elseblock, elselabel); + for i:=0 to blocks.count-1 do - current_asmdata.getjumplabel(pcaseblock(blocks[i])^.blocklabel); + with pcaseblock(blocks[i])^ do + shortcut := GetBranchLabel(statement, blocklabel); with_sign:=is_signed(left.resultdef); if with_sign then @@ -1118,8 +1216,13 @@ { moreover can the size only be appro- } { ximated as it is not known if rel8, } { rel16 or rel32 jumps are used } - max_label:=case_get_max(labels); - labelcnt:=case_count_labels(labels); + labelcnt := 0; + TrueCount := 0; + + CountBoth(labels); + + max_label := case_get_max(labels); + { can we omit the range check of the jump table ? } getrange(left.resultdef,lv,hv); jumptable_no_range:=(lv=min_label) and (hv=max_label); @@ -1128,7 +1231,7 @@ if distv>=0 then dist:=distv.uvalue else - dist:=-distv.svalue; + dist:=aword(-distv.svalue); { optimize for size ? } if cs_opt_size in current_settings.optimizerswitches then @@ -1137,8 +1240,8 @@ (min_label>=int64(low(aint))) and (max_label<=high(aint)) and not((labelcnt<=2) or - ((max_label-min_label)<0) or - ((max_label-min_label)>3*labelcnt)) then + (distv.svalue<0) or + (dist>3*TrueCount)) then begin { if the labels less or more a continuum then } genjumptable(labels,min_label.svalue,max_label.svalue); @@ -1151,7 +1254,7 @@ end else begin - max_dist:=4*labelcnt; + max_dist := 4 * TrueCount; if jumptable_no_range then max_linear_list:=4 else @@ -1187,26 +1290,37 @@ end; { generate the instruction blocks } - for i:=0 to blocks.count-1 do + for i:=0 to blocks.count-1 do with pcaseblock(blocks[i])^ do begin - current_asmdata.CurrAsmList.concat(cai_align.create(current_settings.alignment.jumpalign)); - cg.a_label(current_asmdata.CurrAsmList,pcaseblock(blocks[i])^.blocklabel); - secondpass(pcaseblock(blocks[i])^.statement); - { don't come back to case line } - current_filepos:=current_asmdata.CurrAsmList.getlasttaifilepos^; + { If the labels are not equal, then the block label has been shortcut to point elsewhere, + so there's no need to implement it } + if not shortcut then + begin + current_asmdata.CurrAsmList.concat(cai_align.create(current_settings.alignment.jumpalign)); + cg.a_label(current_asmdata.CurrAsmList,blocklabel); + secondpass(statement); + { don't come back to case line } + current_filepos:=current_asmdata.CurrAsmList.getlasttaifilepos^; {$ifdef OLDREGVARS} - load_all_regvars(current_asmdata.CurrAsmList); + load_all_regvars(current_asmdata.CurrAsmList); {$endif OLDREGVARS} - hlcg.a_jmp_always(current_asmdata.CurrAsmList,endlabel); + hlcg.a_jmp_always(current_asmdata.CurrAsmList,endlabel); + end; end; - current_asmdata.CurrAsmList.concat(cai_align.create(current_settings.alignment.jumpalign)); + { ...and the else block } - hlcg.a_label(current_asmdata.CurrAsmList,elselabel); - if assigned(elseblock) then + if not ShortcutElse then begin - secondpass(elseblock); + current_asmdata.CurrAsmList.concat(cai_align.create(current_settings.alignment.jumpalign)); + hlcg.a_label(current_asmdata.CurrAsmList,elselabel); + end; + + if Assigned(elseblock) then + begin + + secondpass(elseblock); {$ifdef OLDREGVARS} - load_all_regvars(current_asmdata.CurrAsmList); + load_all_regvars(current_asmdata.CurrAsmList); {$endif OLDREGVARS} end; Index: compiler/nset.pas =================================================================== --- compiler/nset.pas (revision 40548) +++ compiler/nset.pas (working copy) @@ -62,6 +62,12 @@ { label (only used in pass_generate_code) } blocklabel : tasmlabel; + { shortcut - set to true if blocklabel isn't actually unique to the + case block due to one of the following conditions: + - if the node contains a jump, then the label is set to that jump's destination, + - if the node is empty, the label is set to the end label. } + shortcut: Boolean; + statementlabel : tlabelnode; { instructions } statement : tnode; @@ -121,6 +127,9 @@ { counts the labels } function case_count_labels(root : pcaselabel) : longint; + { Returns the true count in a case block, which includes each individual + value in a range (e.g. "0..2" counts as 3) } + function case_true_count(root : pcaselabel) : longint; { searches the highest label } function case_get_max(root : pcaselabel) : tconstexprint; { searches the lowest label } @@ -439,6 +448,29 @@ end; + { Returns the true count in a case block, which includes each individual + value in a range (e.g. "0..2" counts as 3) } + function case_true_count(root : pcaselabel) : longint; + var + _l : longint; + + procedure count(p : pcaselabel); + begin + inc(_l, (p^._high.svalue - p^._low.svalue) + 1); + if assigned(p^.less) then + count(p^.less); + if assigned(p^.greater) then + count(p^.greater); + end; + + begin + _l:=0; + count(root); + case_true_count:=_l; + end; + + + function case_get_max(root : pcaselabel) : tconstexprint; var hp : pcaselabel; Index: compiler/x86/nx86set.pas =================================================================== --- compiler/x86/nx86set.pas (revision 40548) +++ compiler/x86/nx86set.pas (working copy) @@ -47,7 +47,7 @@ uses systems, verbose,globals, - symconst,symdef,defutil, + symconst,symdef,defutil,cutils, aasmbase,aasmtai,aasmdata,aasmcpu, cgbase,pass_2,tgobj, ncon, @@ -76,7 +76,13 @@ opcgsize: tcgsize; jumpreg: tregister; labeltyp: taiconst_type; + AlmostExhaustive: Boolean; + lv, hv: TConstExprInt; + ExhaustiveLimit, Range, x, oldmin : aint; + const + ExhaustiveLimitBase = 32; + procedure genitem(list:TAsmList;t : pcaselabel); var i : aint; @@ -83,6 +89,7 @@ begin if assigned(t^.less) then genitem(list,t^.less); + { fill possible hole } i:=last.svalue+1; while i<=t^._low.svalue-1 do @@ -106,16 +113,44 @@ { This generates near pointers on i8086 } labeltyp:=aitconst_ptr; opcgsize:=def_cgsize(opsize); + + AlmostExhaustive := False; + if not(jumptable_no_range) then begin - { a <= x <= b <-> unsigned(x-a) <= (b-a) } - cg.a_op_const_reg(current_asmdata.CurrAsmList,OP_SUB,opcgsize,aint(min_),hregister); - { case expr greater than max_ => goto elselabel } - cg.a_cmp_const_reg_label(current_asmdata.CurrAsmList,opcgsize,OC_A,aint(max_)-aint(min_),hregister,elselabel); - min_:=0; - { do not sign extend when we load the index register, as we applied an offset above } - opcgsize:=tcgsize2unsigned[opcgsize]; + + getrange(left.resultdef,lv,hv); + Range := aint(max_)-aint(min_); + + if (cs_opt_size in current_settings.optimizerswitches) then + { Limit size of jump tables for small enumerations so they have + to be at least two-thirds full before being considered for the + "almost exhaustive" treatment } + ExhaustiveLimit := min(ExhaustiveLimitBase, TrueCount shl 1) + else + ExhaustiveLimit := ExhaustiveLimitBase; + + { If true, then this indicates that almost every possible value of x is covered by + a label. As such, it's more cost-efficient to remove the initial range check and + instead insert the remaining values into the jump table, pointing at elselabel. [Kit] } + if ((hv - lv) - Range <= ExhaustiveLimit) then + begin + oldmin := min_; + min_ := lv.svalue; + AlmostExhaustive := True; + end + else + begin + { a <= x <= b <-> unsigned(x-a) <= (b-a) } + cg.a_op_const_reg(current_asmdata.CurrAsmList,OP_SUB,opcgsize,aint(min_),hregister); + { case expr greater than max_ => goto elselabel } + cg.a_cmp_const_reg_label(current_asmdata.CurrAsmList,opcgsize,OC_A,aint(max_)-aint(min_),hregister,elselabel); + min_:=0; + { do not sign extend when we load the index register, as we applied an offset above } + opcgsize:=tcgsize2unsigned[opcgsize]; + end; end; + current_asmdata.getglobaldatalabel(table); { make it a 32bit register } indexreg:=cg.makeregsize(current_asmdata.CurrAsmList,hregister,OS_INT); @@ -148,7 +183,32 @@ jtlist:=current_procinfo.aktlocaldata; new_section(jtlist,sec_rodata,current_procinfo.procdef.mangledname,sizeof(aint)); jtlist.concat(Tai_label.Create(table)); - genitem(jtlist,hp); + + if AlmostExhaustive then + begin + { Fill the table with the values below _min } + x := lv.svalue; + while x < oldmin do + begin + jtlist.concat(Tai_const.Create_type_sym(labeltyp, elselabel)); + Inc(x); + end; + + genitem(jtlist,hp); + + { Fill the table with the values above _max } + { Subtracting one from hv and not adding 1 to max averts the risk of an overflow } + x := max_; + hv := hv - 1; + while x <= hv.svalue do + begin + jtlist.concat(Tai_const.Create_type_sym(labeltyp, elselabel)); + Inc(x); + end; + + end + else + genitem(jtlist,hp) end; @@ -251,7 +311,7 @@ begin last:=0; lastrange:=false; - first:=true; + first:=(labelcnt > 1); { Can greatly simplify the range checks if there's only one label } genitem(hp); cg.a_jmp_always(current_asmdata.CurrAsmList,elselabel); end; Index: compiler/x86_64/nx64set.pas =================================================================== --- compiler/x86_64/nx64set.pas (revision 40548) +++ compiler/x86_64/nx64set.pas (working copy) @@ -26,6 +26,7 @@ interface uses + constexp, globtype, nset,nx86set; @@ -39,15 +40,21 @@ implementation uses - systems, - verbose,globals,constexp, - defutil, - aasmbase,aasmtai,aasmdata, + systems,cpuinfo, + verbose,globals, + defutil,cutils, + aasmbase,aasmtai,aasmdata,aasmcpu, cgbase, cpubase,procinfo, - cga,cgutils,cgobj; + cga,cgutils,cgobj,cgx86; + function CreateCMOVInstr(cond : TAsmCond; size: TOpSize; reg1, reg2: TRegister): taicpu; + begin + Result := taicpu.op_reg_reg(A_CMOVcc, size, reg1, reg2); + Result.condition := cond; + end; + {***************************************************************************** TX8664CASENODE *****************************************************************************} @@ -55,6 +62,7 @@ procedure tx8664casenode.optimizevalues(var max_linear_list:aint;var max_dist:aword); begin inc(max_linear_list,9); + max_dist := min(max_dist,2048); { Don't allow jump tables to get too large } end; @@ -66,32 +74,39 @@ tablelabel: TAsmLabel; basereg,indexreg,jumpreg: TRegister; href: TReference; + jtlist: TAsmList; opcgsize: tcgsize; sectype: TAsmSectiontype; jtitemconsttype: taiconst_type; + AlmostExhaustive: Boolean; + lv, hv: TConstExprInt; + ExhaustiveLimit, Range, x, oldmin : aint; - procedure genitem(list:TAsmList;t : pcaselabel); + const + ExhaustiveLimitBase = 32; + + procedure genitem(t : pcaselabel); var i : aint; begin if assigned(t^.less) then - genitem(list,t^.less); + genitem(t^.less); { fill possible hole } i:=last.svalue+1; while i<=t^._low.svalue-1 do begin - list.concat(Tai_const.Create_rel_sym(jtitemconsttype,tablelabel,elselabel)); + jtlist.concat(Tai_const.Create_rel_sym(jtitemconsttype,tablelabel,elselabel)); inc(i); end; i:=t^._low.svalue; while i<=t^._high.svalue do begin - list.concat(Tai_const.Create_rel_sym(jtitemconsttype,tablelabel,blocklabel(t^.blockid))); + jtlist.concat(Tai_const.Create_rel_sym(jtitemconsttype,tablelabel,blocklabel(t^.blockid))); inc(i); end; last:=t^._high; if assigned(t^.greater) then - genitem(list,t^.greater); + genitem(t^.greater); end; begin @@ -101,38 +116,67 @@ { see https://gmplib.org/list-archives/gmp-bugs/2012-December/002836.html } jtitemconsttype:=aitconst_darwin_dwarf_delta32; + jtlist := current_asmdata.CurrAsmList; last:=min_; opcgsize:=def_cgsize(opsize); + + AlmostExhaustive := False; + oldmin := min_; + if not(jumptable_no_range) then begin - { a <= x <= b <-> unsigned(x-a) <= (b-a) } - cg.a_op_const_reg(current_asmdata.CurrAsmList,OP_SUB,opcgsize,aint(min_),hregister); - { case expr greater than max_ => goto elselabel } - cg.a_cmp_const_reg_label(current_asmdata.CurrAsmList,opcgsize,OC_A,aint(max_)-aint(min_),hregister,elselabel); - min_:=0; - { do not sign extend when we load the index register, as we applied an offset above } - opcgsize:=tcgsize2unsigned[opcgsize]; + + getrange(left.resultdef,lv,hv); + Range := aint(max_)-aint(min_); + + if (cs_opt_size in current_settings.optimizerswitches) then + { Limit size of jump tables for small enumerations so they have + to be at least two-thirds full before being considered for the + "almost exhaustive" treatment } + ExhaustiveLimit := min(ExhaustiveLimitBase, TrueCount shl 1) + else + ExhaustiveLimit := ExhaustiveLimitBase; + + { If true, then this indicates that almost every possible value of x is covered by + a label. As such, it's more cost-efficient to remove the initial range check and + instead insert the remaining values into the jump table, pointing at elselabel. [Kit] } + if ((hv - lv) - Range <= ExhaustiveLimit) then + begin + oldmin := min_; + min_ := lv.svalue; + AlmostExhaustive := True; + end + else + begin + { a <= x <= b <-> unsigned(x-a) <= (b-a) } + cg.a_op_const_reg(jtlist,OP_SUB,opcgsize,aint(min_),hregister); + { case expr greater than max_ => goto elselabel } + cg.a_cmp_const_reg_label(jtlist,opcgsize,OC_A,Range,hregister,elselabel); + min_:=0; + { do not sign extend when we load the index register, as we applied an offset above } + opcgsize:=tcgsize2unsigned[opcgsize]; + end; end; { local label in order to avoid using GOT } current_asmdata.getlabel(tablelabel,alt_data); - indexreg:=cg.makeregsize(current_asmdata.CurrAsmList,hregister,OS_ADDR); - cg.a_load_reg_reg(current_asmdata.CurrAsmList,opcgsize,OS_ADDR,hregister,indexreg); + indexreg:=cg.makeregsize(jtlist,hregister,OS_ADDR); + cg.a_load_reg_reg(jtlist,opcgsize,OS_ADDR,hregister,indexreg); { load table address } reference_reset_symbol(href,tablelabel,0,4,[]); - basereg:=cg.getaddressregister(current_asmdata.CurrAsmList); - cg.a_loadaddr_ref_reg(current_asmdata.CurrAsmList,href,basereg); + basereg:=cg.getaddressregister(jtlist); + cg.a_loadaddr_ref_reg(jtlist,href,basereg); { load table slot, 32-bit sign extended } reference_reset_base(href,basereg,-aint(min_)*4,ctempposinvalid,4,[]); href.index:=indexreg; href.scalefactor:=4; - jumpreg:=cg.getaddressregister(current_asmdata.CurrAsmList); - cg.a_load_ref_reg(current_asmdata.CurrAsmList,OS_S32,OS_ADDR,href,jumpreg); + jumpreg:=cg.getaddressregister(jtlist); + cg.a_load_ref_reg(jtlist,OS_S32,OS_ADDR,href,jumpreg); { add table address } reference_reset_base(href,basereg,0,ctempposinvalid,sizeof(pint),[]); href.index:=jumpreg; href.scalefactor:=1; - cg.a_loadaddr_ref_reg(current_asmdata.CurrAsmList,href,jumpreg); + cg.a_loadaddr_ref_reg(jtlist,href,jumpreg); { and finally jump } emit_reg(A_JMP,S_NO,jumpreg); { generate jump table } @@ -151,9 +195,36 @@ is inserted right after the routine, it will become part of the same subsection that contains the routine's code } sectype:=sec_code; - new_section(current_procinfo.aktlocaldata,sectype,current_procinfo.procdef.mangledname,4); - current_procinfo.aktlocaldata.concat(Tai_label.Create(tablelabel)); - genitem(current_procinfo.aktlocaldata,hp); + + jtlist := current_procinfo.aktlocaldata; + new_section(jtlist,sectype,current_procinfo.procdef.mangledname,4); + jtlist.concat(Tai_label.Create(tablelabel)); + + if AlmostExhaustive then + begin + { Fill the table with the values below _min } + x := lv.svalue; + while x < oldmin do + begin + jtlist.concat(Tai_const.Create_rel_sym(jtitemconsttype,tablelabel,elselabel)); + Inc(x); + end; + + genitem(hp); + + { Fill the table with the values above _max } + { Subtracting one from hv and not adding 1 to max_ averts the risk of an overflow } + x := max_; + hv := hv - 1; + while x <= hv.svalue do + begin + jtlist.concat(Tai_const.Create_rel_sym(jtitemconsttype,tablelabel,elselabel)); + Inc(x); + end; + + end + else + genitem(hp); end; begin