This is an automated email from the ASF dual-hosted git repository. sebwrede pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/systemds.git
The following commit(s) were added to refs/heads/main by this push: new 5ed7368 [MINOR] Move IPAPassRewriteFederatedPlan to IPA Package 5ed7368 is described below commit 5ed73681ddad939b526027395d017473ccc8a1f1 Author: sebwrede <swr...@know-center.at> AuthorDate: Mon Nov 8 12:44:38 2021 +0100 [MINOR] Move IPAPassRewriteFederatedPlan to IPA Package The IPAPassRewriteFederatedPlan is moved to the IPA package where it belongs. Additionally, the code is auto-formatted. Closes #1445. --- .../hops/rewrite/IPAPassRewriteFederatedPlan.html | 2 +- .../class-use/IPAPassRewriteFederatedPlan.html | 8 +- .../IPAPassRewriteFederatedPlan.java | 124 +++++++++++---------- .../sysds/hops/ipa/InterProceduralAnalysis.java | 1 - 4 files changed, 71 insertions(+), 64 deletions(-) diff --git a/docs/api/java/org/apache/sysds/hops/rewrite/IPAPassRewriteFederatedPlan.html b/docs/api/java/org/apache/sysds/hops/rewrite/IPAPassRewriteFederatedPlan.html index b7039b9..21bb921 100644 --- a/docs/api/java/org/apache/sysds/hops/rewrite/IPAPassRewriteFederatedPlan.html +++ b/docs/api/java/org/apache/sysds/hops/rewrite/IPAPassRewriteFederatedPlan.html @@ -103,7 +103,7 @@ var activeTableTab = "activeTableTab"; <li><a href="../../../../../org/apache/sysds/hops/ipa/IPAPass.html" title="class in org.apache.sysds.hops.ipa">org.apache.sysds.hops.ipa.IPAPass</a></li> <li> <ul class="inheritance"> -<li>org.apache.sysds.hops.rewrite.IPAPassRewriteFederatedPlan</li> +<li>org.apache.sysds.hops.ipa.IPAPassRewriteFederatedPlan</li> </ul> </li> </ul> diff --git a/docs/api/java/org/apache/sysds/hops/rewrite/class-use/IPAPassRewriteFederatedPlan.html b/docs/api/java/org/apache/sysds/hops/rewrite/class-use/IPAPassRewriteFederatedPlan.html index 3218937..c31c389 100644 --- a/docs/api/java/org/apache/sysds/hops/rewrite/class-use/IPAPassRewriteFederatedPlan.html +++ b/docs/api/java/org/apache/sysds/hops/rewrite/class-use/IPAPassRewriteFederatedPlan.html @@ -4,7 +4,7 @@ <head> <!-- Generated by javadoc --> <meta http-equiv="Content-Type" content="text/html; charset=UTF-8"> -<title>Uses of Class org.apache.sysds.hops.rewrite.IPAPassRewriteFederatedPlan (SystemDS 2.3.0-SNAPSHOT API)</title> +<title>Uses of Class org.apache.sysds.hops.ipa.IPAPassRewriteFederatedPlan (SystemDS 2.3.0-SNAPSHOT API)</title> <link rel="stylesheet" type="text/css" href="../../../../../../stylesheet.css" title="Style"> <script type="text/javascript" src="../../../../../../script.js"></script> </head> @@ -12,7 +12,7 @@ <script type="text/javascript"><!-- try { if (location.href.indexOf('is-external=true') == -1) { - parent.document.title="Uses of Class org.apache.sysds.hops.rewrite.IPAPassRewriteFederatedPlan (SystemDS 2.3.0-SNAPSHOT API)"; + parent.document.title="Uses of Class org.apache.sysds.hops.ipa.IPAPassRewriteFederatedPlan (SystemDS 2.3.0-SNAPSHOT API)"; } } catch(err) { @@ -70,9 +70,9 @@ </a></div> <!-- ========= END OF TOP NAVBAR ========= --> <div class="header"> -<h2 title="Uses of Class org.apache.sysds.hops.rewrite.IPAPassRewriteFederatedPlan" class="title">Uses of Class<br>org.apache.sysds.hops.rewrite.IPAPassRewriteFederatedPlan</h2> +<h2 title="Uses of Class org.apache.sysds.hops.ipa.IPAPassRewriteFederatedPlan" class="title">Uses of Class<br>org.apache.sysds.hops.ipa.IPAPassRewriteFederatedPlan</h2> </div> -<div class="classUseContainer">No usage of org.apache.sysds.hops.rewrite.IPAPassRewriteFederatedPlan</div> +<div class="classUseContainer">No usage of org.apache.sysds.hops.ipa.IPAPassRewriteFederatedPlan</div> <!-- ======= START OF BOTTOM NAVBAR ====== --> <div class="bottomNav"><a name="navbar.bottom"> <!-- --> diff --git a/src/main/java/org/apache/sysds/hops/rewrite/IPAPassRewriteFederatedPlan.java b/src/main/java/org/apache/sysds/hops/ipa/IPAPassRewriteFederatedPlan.java similarity index 79% rename from src/main/java/org/apache/sysds/hops/rewrite/IPAPassRewriteFederatedPlan.java rename to src/main/java/org/apache/sysds/hops/ipa/IPAPassRewriteFederatedPlan.java index 2a28399..8c8df49 100644 --- a/src/main/java/org/apache/sysds/hops/rewrite/IPAPassRewriteFederatedPlan.java +++ b/src/main/java/org/apache/sysds/hops/ipa/IPAPassRewriteFederatedPlan.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.sysds.hops.rewrite; +package org.apache.sysds.hops.ipa; import org.apache.sysds.api.DMLException; import org.apache.sysds.hops.AggBinaryOp; @@ -30,9 +30,6 @@ import org.apache.sysds.hops.OptimizerUtils; import org.apache.sysds.hops.ReorgOp; import org.apache.sysds.hops.TernaryOp; import org.apache.sysds.hops.cost.HopRel; -import org.apache.sysds.hops.ipa.FunctionCallGraph; -import org.apache.sysds.hops.ipa.FunctionCallSizeInfo; -import org.apache.sysds.hops.ipa.IPAPass; import org.apache.sysds.parser.DMLProgram; import org.apache.sysds.parser.ForStatement; import org.apache.sysds.parser.ForStatementBlock; @@ -69,8 +66,7 @@ public class IPAPassRewriteFederatedPlan extends IPAPass { * @param fgraph function call graph * @return true if federated compilation is activated. */ - @Override - public boolean isApplicable(FunctionCallGraph fgraph) { + @Override public boolean isApplicable(FunctionCallGraph fgraph) { return OptimizerUtils.FEDERATED_COMPILATION; } @@ -83,8 +79,8 @@ public class IPAPassRewriteFederatedPlan extends IPAPass { * @param fcallSizes function call size infos * @return false since the function call graph never has to be rebuilt */ - @Override - public boolean rewriteProgram(DMLProgram prog, FunctionCallGraph fgraph, FunctionCallSizeInfo fcallSizes) { + @Override public boolean rewriteProgram(DMLProgram prog, FunctionCallGraph fgraph, + FunctionCallSizeInfo fcallSizes) { rewriteStatementBlocks(prog, prog.getStatementBlocks()); return false; } @@ -95,12 +91,12 @@ public class IPAPassRewriteFederatedPlan extends IPAPass { * The method calls the contained statement blocks recursively. * * @param prog dml program - * @param sbs list of statement blocks + * @param sbs list of statement blocks * @return list of statement blocks with the federated output value updated for each hop */ public ArrayList<StatementBlock> rewriteStatementBlocks(DMLProgram prog, List<StatementBlock> sbs) { ArrayList<StatementBlock> rewrittenStmBlocks = new ArrayList<>(); - for ( StatementBlock stmBlock : sbs ) + for(StatementBlock stmBlock : sbs) rewrittenStmBlocks.addAll(rewriteStatementBlock(prog, stmBlock)); return rewrittenStmBlocks; } @@ -111,19 +107,19 @@ public class IPAPassRewriteFederatedPlan extends IPAPass { * The method calls the contained statement blocks recursively. * * @param prog dml program - * @param sb statement block + * @param sb statement block * @return list of statement blocks with the federated output value updated for each hop */ public ArrayList<StatementBlock> rewriteStatementBlock(DMLProgram prog, StatementBlock sb) { - if ( sb instanceof WhileStatementBlock) + if(sb instanceof WhileStatementBlock) return rewriteWhileStatementBlock(prog, (WhileStatementBlock) sb); - else if ( sb instanceof IfStatementBlock) + else if(sb instanceof IfStatementBlock) return rewriteIfStatementBlock(prog, (IfStatementBlock) sb); - else if ( sb instanceof ForStatementBlock){ + else if(sb instanceof ForStatementBlock) { // This also includes ParForStatementBlocks return rewriteForStatementBlock(prog, (ForStatementBlock) sb); } - else if ( sb instanceof FunctionStatementBlock) + else if(sb instanceof FunctionStatementBlock) return rewriteFunctionStatementBlock(prog, (FunctionStatementBlock) sb); else { // StatementBlock type (no subclass) @@ -131,19 +127,19 @@ public class IPAPassRewriteFederatedPlan extends IPAPass { } } - private ArrayList<StatementBlock> rewriteWhileStatementBlock(DMLProgram prog, WhileStatementBlock whileSB){ + private ArrayList<StatementBlock> rewriteWhileStatementBlock(DMLProgram prog, WhileStatementBlock whileSB) { Hop whilePredicateHop = whileSB.getPredicateHops(); selectFederatedExecutionPlan(whilePredicateHop); - for ( Statement stm : whileSB.getStatements() ){ + for(Statement stm : whileSB.getStatements()) { WhileStatement whileStm = (WhileStatement) stm; whileStm.setBody(rewriteStatementBlocks(prog, whileStm.getBody())); } return new ArrayList<>(Collections.singletonList(whileSB)); } - private ArrayList<StatementBlock> rewriteIfStatementBlock(DMLProgram prog, IfStatementBlock ifSB){ + private ArrayList<StatementBlock> rewriteIfStatementBlock(DMLProgram prog, IfStatementBlock ifSB) { selectFederatedExecutionPlan(ifSB.getPredicateHops()); - for ( Statement statement : ifSB.getStatements() ){ + for(Statement statement : ifSB.getStatements()) { IfStatement ifStatement = (IfStatement) statement; ifStatement.setIfBody(rewriteStatementBlocks(prog, ifStatement.getIfBody())); ifStatement.setElseBody(rewriteStatementBlocks(prog, ifStatement.getElseBody())); @@ -151,34 +147,35 @@ public class IPAPassRewriteFederatedPlan extends IPAPass { return new ArrayList<>(Collections.singletonList(ifSB)); } - private ArrayList<StatementBlock> rewriteForStatementBlock(DMLProgram prog, ForStatementBlock forSB){ + private ArrayList<StatementBlock> rewriteForStatementBlock(DMLProgram prog, ForStatementBlock forSB) { selectFederatedExecutionPlan(forSB.getFromHops()); selectFederatedExecutionPlan(forSB.getToHops()); selectFederatedExecutionPlan(forSB.getIncrementHops()); - for ( Statement statement : forSB.getStatements() ){ - ForStatement forStatement = ((ForStatement)statement); + for(Statement statement : forSB.getStatements()) { + ForStatement forStatement = ((ForStatement) statement); forStatement.setBody(rewriteStatementBlocks(prog, forStatement.getBody())); } return new ArrayList<>(Collections.singletonList(forSB)); } - private ArrayList<StatementBlock> rewriteFunctionStatementBlock(DMLProgram prog, FunctionStatementBlock funcSB){ - for ( Statement statement : funcSB.getStatements() ){ + private ArrayList<StatementBlock> rewriteFunctionStatementBlock(DMLProgram prog, FunctionStatementBlock funcSB) { + for(Statement statement : funcSB.getStatements()) { FunctionStatement funcStm = (FunctionStatement) statement; funcStm.setBody(rewriteStatementBlocks(prog, funcStm.getBody())); } return new ArrayList<>(Collections.singletonList(funcSB)); } - private ArrayList<StatementBlock> rewriteDefaultStatementBlock(DMLProgram prog, StatementBlock sb){ - if ( sb.getHops() != null && !sb.getHops().isEmpty() ){ - for ( Hop sbHop : sb.getHops() ){ - if ( sbHop instanceof FunctionOp ){ + private ArrayList<StatementBlock> rewriteDefaultStatementBlock(DMLProgram prog, StatementBlock sb) { + if(sb.getHops() != null && !sb.getHops().isEmpty()) { + for(Hop sbHop : sb.getHops()) { + if(sbHop instanceof FunctionOp) { String funcName = ((FunctionOp) sbHop).getFunctionName(); FunctionStatementBlock sbFuncBlock = prog.getBuiltinFunctionDictionary().getFunction(funcName); rewriteStatementBlock(prog, sbFuncBlock); } - else selectFederatedExecutionPlan(sbHop); + else + selectFederatedExecutionPlan(sbHop); } } return new ArrayList<>(Collections.singletonList(sb)); @@ -188,40 +185,45 @@ public class IPAPassRewriteFederatedPlan extends IPAPass { * Sets FederatedOutput field of all hops in DAG starting from given root. * The FederatedOutput chosen for root is the minimum cost HopRel found in memo table for the given root. * The FederatedOutput values chosen for the inputs to the root are chosen based on the input dependencies. + * * @param root hop for which FederatedOutput needs to be set */ - private void setFinalFedout(Hop root){ - HopRel optimalRootHopRel = hopRelMemo.get(root.getHopID()).stream().min(Comparator.comparingDouble(HopRel::getCost)) + private void setFinalFedout(Hop root) { + HopRel optimalRootHopRel = hopRelMemo.get(root.getHopID()).stream() + .min(Comparator.comparingDouble(HopRel::getCost)) .orElseThrow(() -> new DMLException("Hop root " + root + " has no feasible federated output alternatives")); setFinalFedout(root, optimalRootHopRel); } /** * Update the FederatedOutput value and cost based on information stored in given rootHopRel. - * @param root hop for which FederatedOutput is set + * + * @param root hop for which FederatedOutput is set * @param rootHopRel from which FederatedOutput value and cost is retrieved */ - private void setFinalFedout(Hop root, HopRel rootHopRel){ + private void setFinalFedout(Hop root, HopRel rootHopRel) { updateFederatedOutput(root, rootHopRel); visitInputDependency(rootHopRel); } /** * Sets FederatedOutput value for each of the inputs of rootHopRel + * * @param rootHopRel which has its input values updated */ - private void visitInputDependency(HopRel rootHopRel){ + private void visitInputDependency(HopRel rootHopRel) { List<HopRel> hopRelInputs = rootHopRel.getInputDependency(); - for ( HopRel input : hopRelInputs ) + for(HopRel input : hopRelInputs) setFinalFedout(input.getHopRef(), input); } /** * Updates FederatedOutput value and cost estimate based on updateHopRel values. - * @param root which has its values updated + * + * @param root which has its values updated * @param updateHopRel from which the values are retrieved */ - private void updateFederatedOutput(Hop root, HopRel updateHopRel){ + private void updateFederatedOutput(Hop root, HopRel updateHopRel) { root.setFederatedOutput(updateHopRel.getFederatedOutput()); root.setFederatedCost(updateHopRel.getCostObject()); } @@ -229,6 +231,7 @@ public class IPAPassRewriteFederatedPlan extends IPAPass { /** * Select federated execution plan for every Hop in the DAG starting from given roots. * The cost estimates of the hops are also updated when FederatedOutput is updated in the hops. + * * @param roots starting point for going through the Hop DAG to update the FederatedOutput fields. */ @SuppressWarnings("unused") @@ -239,57 +242,61 @@ public class IPAPassRewriteFederatedPlan extends IPAPass { /** * Select federated execution plan for every Hop in the DAG starting from given root. + * * @param root starting point for going through the Hop DAG to update the federatedOutput fields */ - private void selectFederatedExecutionPlan(Hop root){ + private void selectFederatedExecutionPlan(Hop root) { visitFedPlanHop(root); setFinalFedout(root); } /** * Go through the Hop DAG and set the FederatedOutput field and cost estimate for each Hop from leaf to given currentHop. + * * @param currentHop the Hop from which the DAG is visited */ - private void visitFedPlanHop(Hop currentHop){ + private void visitFedPlanHop(Hop currentHop) { // If the currentHop is in the hopRelMemo table, it means that it has been visited - if ( hopRelMemo.containsKey(currentHop.getHopID()) ) + if(hopRelMemo.containsKey(currentHop.getHopID())) return; // If the currentHop has input, then the input should be visited depth-first - if ( currentHop.getInput() != null && currentHop.getInput().size() > 0 ){ - for ( Hop input : currentHop.getInput() ) + if(currentHop.getInput() != null && currentHop.getInput().size() > 0) { + for(Hop input : currentHop.getInput()) visitFedPlanHop(input); } // Put FOUT, LOUT, and None HopRels into the memo table ArrayList<HopRel> hopRels = new ArrayList<>(); - if ( isFedInstSupportedHop(currentHop) ){ - for ( FEDInstruction.FederatedOutput fedoutValue : FEDInstruction.FederatedOutput.values() ) - if ( isFedOutSupported(currentHop, fedoutValue) ) - hopRels.add(new HopRel(currentHop,fedoutValue, hopRelMemo)); + if(isFedInstSupportedHop(currentHop)) { + for(FEDInstruction.FederatedOutput fedoutValue : FEDInstruction.FederatedOutput.values()) + if(isFedOutSupported(currentHop, fedoutValue)) + hopRels.add(new HopRel(currentHop, fedoutValue, hopRelMemo)); } - if ( hopRels.isEmpty() ) + if(hopRels.isEmpty()) hopRels.add(new HopRel(currentHop, FEDInstruction.FederatedOutput.NONE, hopRelMemo)); hopRelMemo.put(currentHop.getHopID(), hopRels); } /** * Checks if the instructions related to the given hop supports FOUT/LOUT processing. + * * @param hop to check for federated support * @return true if federated instructions related to hop supports FOUT/LOUT processing */ - private boolean isFedInstSupportedHop(Hop hop){ + private boolean isFedInstSupportedHop(Hop hop) { // The following operations are supported given that the above conditions have not returned already - return ( hop instanceof AggBinaryOp || hop instanceof BinaryOp || hop instanceof ReorgOp + return (hop instanceof AggBinaryOp || hop instanceof BinaryOp || hop instanceof ReorgOp || hop instanceof AggUnaryOp || hop instanceof TernaryOp || hop instanceof DataOp); } /** * Checks if the associatedHop supports the given federated output value. + * * @param associatedHop to check support of - * @param fedOut federated output value + * @param fedOut federated output value * @return true if associatedHop supports fedOut */ - private boolean isFedOutSupported(Hop associatedHop, FEDInstruction.FederatedOutput fedOut){ - switch(fedOut){ + private boolean isFedOutSupported(Hop associatedHop, FEDInstruction.FederatedOutput fedOut) { + switch(fedOut) { case FOUT: return isFOUTSupported(associatedHop); case LOUT: @@ -303,17 +310,17 @@ public class IPAPassRewriteFederatedPlan extends IPAPass { /** * Checks to see if the associatedHop supports FOUT. + * * @param associatedHop for which FOUT support is checked * @return true if FOUT is supported by the associatedHop */ - private boolean isFOUTSupported(Hop associatedHop){ + private boolean isFOUTSupported(Hop associatedHop) { // If the output of AggUnaryOp is a scalar, the operation cannot be FOUT - if ( associatedHop instanceof AggUnaryOp && associatedHop.isScalar() ) + if(associatedHop instanceof AggUnaryOp && associatedHop.isScalar()) return false; // It can only be FOUT if at least one of the inputs are FOUT, except if it is a federated DataOp - if ( associatedHop.getInput().stream().noneMatch( - input -> hopRelMemo.get(input.getHopID()).stream().anyMatch(HopRel::hasFederatedOutput) ) - && !associatedHop.isFederatedDataOp() ) + if(associatedHop.getInput().stream().noneMatch(input -> hopRelMemo.get(input.getHopID()).stream() + .anyMatch(HopRel::hasFederatedOutput)) && !associatedHop.isFederatedDataOp()) return false; return true; } @@ -321,10 +328,11 @@ public class IPAPassRewriteFederatedPlan extends IPAPass { /** * Checks to see if the associatedHop supports LOUT. * It supports LOUT if the output has no privacy constraints. + * * @param associatedHop for which LOUT support is checked. * @return true if LOUT is supported by the associatedHop */ - private boolean isLOUTSupported(Hop associatedHop){ + private boolean isLOUTSupported(Hop associatedHop) { return associatedHop.getPrivacy() == null || !associatedHop.getPrivacy().hasConstraints(); } } diff --git a/src/main/java/org/apache/sysds/hops/ipa/InterProceduralAnalysis.java b/src/main/java/org/apache/sysds/hops/ipa/InterProceduralAnalysis.java index b0597eb..20644fa 100644 --- a/src/main/java/org/apache/sysds/hops/ipa/InterProceduralAnalysis.java +++ b/src/main/java/org/apache/sysds/hops/ipa/InterProceduralAnalysis.java @@ -34,7 +34,6 @@ import org.apache.sysds.hops.HopsException; import org.apache.sysds.hops.LiteralOp; import org.apache.sysds.hops.OptimizerUtils; import org.apache.sysds.hops.recompile.Recompiler; -import org.apache.sysds.hops.rewrite.IPAPassRewriteFederatedPlan; import org.apache.sysds.parser.DMLProgram; import org.apache.sysds.parser.DMLTranslator; import org.apache.sysds.parser.DataIdentifier;