This is an automated email from the ASF dual-hosted git repository.
arnabp20 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/systemds.git
The following commit(s) were added to refs/heads/master by this push:
new 76a7962 Move Privacy Post-Processing To Be Before Lineage Cache
76a7962 is described below
commit 76a79629db740572a0f7af116fce5afddc9f4f28
Author: sebwrede <[email protected]>
AuthorDate: Wed Oct 14 13:10:25 2020 +0200
Move Privacy Post-Processing To Be Before Lineage Cache
---
.../sysds/runtime/controlprogram/ProgramBlock.java | 4 +
.../controlprogram/caching/CacheableData.java | 1 +
.../sysds/runtime/instructions/Instruction.java | 5 +-
.../privacy/propagation/PrivacyPropagator.java | 17 ++++-
.../test/functions/privacy/PrivacyLineageTest.java | 86 ++++++++++++++++++++++
.../functions/privacy/PrivacyLineageTest.dml | 28 +++++++
6 files changed, 136 insertions(+), 5 deletions(-)
diff --git
a/src/main/java/org/apache/sysds/runtime/controlprogram/ProgramBlock.java
b/src/main/java/org/apache/sysds/runtime/controlprogram/ProgramBlock.java
index b9ef965..bb34bf6 100644
--- a/src/main/java/org/apache/sysds/runtime/controlprogram/ProgramBlock.java
+++ b/src/main/java/org/apache/sysds/runtime/controlprogram/ProgramBlock.java
@@ -45,6 +45,7 @@ import org.apache.sysds.runtime.instructions.cp.StringObject;
import org.apache.sysds.runtime.lineage.LineageCache;
import org.apache.sysds.runtime.lineage.LineageCacheConfig.ReuseCacheType;
import org.apache.sysds.runtime.matrix.data.MatrixBlock;
+import org.apache.sysds.runtime.privacy.propagation.PrivacyPropagator;
import org.apache.sysds.utils.Statistics;
import java.util.ArrayList;
@@ -260,6 +261,9 @@ public abstract class ProgramBlock implements ParseInfo
}
}
+ // propagate input privacy constraints to output
+ PrivacyPropagator.postProcessInstruction(tmp, ec);
+
// optional trace information (instruction and runtime)
if( LOG.isTraceEnabled() ) {
long t1 = System.nanoTime();
diff --git
a/src/main/java/org/apache/sysds/runtime/controlprogram/caching/CacheableData.java
b/src/main/java/org/apache/sysds/runtime/controlprogram/caching/CacheableData.java
index e5fc8d4..6828c7a 100644
---
a/src/main/java/org/apache/sysds/runtime/controlprogram/caching/CacheableData.java
+++
b/src/main/java/org/apache/sysds/runtime/controlprogram/caching/CacheableData.java
@@ -225,6 +225,7 @@ public abstract class CacheableData<T extends CacheBlock>
extends Data
_hdfsFileName = that._hdfsFileName;
_hdfsFileExists = that._hdfsFileExists;
_gpuObjects = that._gpuObjects;
+ _privacyConstraint = that._privacyConstraint;
}
diff --git
a/src/main/java/org/apache/sysds/runtime/instructions/Instruction.java
b/src/main/java/org/apache/sysds/runtime/instructions/Instruction.java
index 10645ab..7be54e0 100644
--- a/src/main/java/org/apache/sysds/runtime/instructions/Instruction.java
+++ b/src/main/java/org/apache/sysds/runtime/instructions/Instruction.java
@@ -27,7 +27,6 @@ import org.apache.sysds.parser.DataIdentifier;
import org.apache.sysds.runtime.controlprogram.context.ExecutionContext;
import org.apache.sysds.runtime.matrix.operators.Operator;
import org.apache.sysds.runtime.privacy.PrivacyConstraint;
-import org.apache.sysds.runtime.privacy.propagation.PrivacyPropagator;
public abstract class Instruction
{
@@ -251,7 +250,5 @@ public abstract class Instruction
*
* @param ec execution context
*/
- public void postprocessInstruction(ExecutionContext ec) {
- PrivacyPropagator.postProcessInstruction(this, ec);
- }
+ public void postprocessInstruction(ExecutionContext ec) {}
}
diff --git
a/src/main/java/org/apache/sysds/runtime/privacy/propagation/PrivacyPropagator.java
b/src/main/java/org/apache/sysds/runtime/privacy/propagation/PrivacyPropagator.java
index 7b66e87..b6b845d 100644
---
a/src/main/java/org/apache/sysds/runtime/privacy/propagation/PrivacyPropagator.java
+++
b/src/main/java/org/apache/sysds/runtime/privacy/propagation/PrivacyPropagator.java
@@ -150,6 +150,13 @@ public class PrivacyPropagator
return mergedPrivacyConstraint;
}
+ /**
+ * Propagate privacy constraints from input to output CPOperands
+ * in case the privacy constraints of the input are activated.
+ * @param inst instruction for which the privacy constraints are
propagated
+ * @param ec execution context
+ * @return instruction with propagated privacy constraints (usually the
same instance as the input inst)
+ */
public static Instruction preprocessInstruction(Instruction inst,
ExecutionContext ec){
switch ( inst.getType() ){
case CONTROL_PROGRAM:
@@ -284,6 +291,7 @@ public class PrivacyPropagator
MatrixBlock input2 =
ec.getMatrixInput(inst.input2.getName());
Propagator propagator = new
MatrixMultiplicationPropagatorPrivateFirst(input1, privacyConstraint1, input2,
privacyConstraint2);
mergedPrivacyConstraint =
propagator.propagate();
+
ec.releaseMatrixInput(inst.input1.getName(), inst.input2.getName());
}
else {
mergedPrivacyConstraint = mergeNary(new
PrivacyConstraint[]{privacyConstraint1,privacyConstraint2},
@@ -562,13 +570,20 @@ public class PrivacyPropagator
}
}
+ /**
+ * Propagate privacy constraints to output variables
+ * based on privacy constraint of CPOperand output in instruction
+ * which has been set during privacy propagation preprocessing.
+ * @param inst instruction for which privacy constraints are propagated
+ * @param ec execution context
+ */
public static void postProcessInstruction(Instruction inst,
ExecutionContext ec){
// if inst has output
List<CPOperand> instOutputs = getOutputOperands(inst);
if (!instOutputs.isEmpty()){
for ( CPOperand output : instOutputs ){
PrivacyConstraint outputPrivacyConstraint =
output.getPrivacyConstraint();
- if (
privacyConstraintActivated(outputPrivacyConstraint) )
+ if (
privacyConstraintActivated(outputPrivacyConstraint) || (outputPrivacyConstraint
!= null && outputPrivacyConstraint.hasFineGrainedConstraints()))
setOutputPrivacyConstraint(ec,
outputPrivacyConstraint, output.getName());
}
}
diff --git
a/src/test/java/org/apache/sysds/test/functions/privacy/PrivacyLineageTest.java
b/src/test/java/org/apache/sysds/test/functions/privacy/PrivacyLineageTest.java
new file mode 100644
index 0000000..96a1f3e
--- /dev/null
+++
b/src/test/java/org/apache/sysds/test/functions/privacy/PrivacyLineageTest.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysds.test.functions.privacy;
+
+import org.apache.sysds.parser.DataExpression;
+import org.apache.sysds.runtime.meta.MatrixCharacteristics;
+import org.apache.sysds.runtime.privacy.PrivacyConstraint;
+import org.apache.sysds.runtime.privacy.finegrained.DataRange;
+import org.apache.sysds.test.AutomatedTestBase;
+import org.apache.sysds.test.TestConfiguration;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class PrivacyLineageTest extends AutomatedTestBase {
+
+ private static final String TEST_DIR = "functions/privacy/";
+ private final static String TEST_CLASS_DIR = TEST_DIR +
PrivacyLineageTest.class.getSimpleName() + "/";
+
+ @Override public void setUp() {
+ addTestConfiguration("LineageReuse",
+ new TestConfiguration(TEST_CLASS_DIR,
"PrivacyLineageTest", new String[]{"c"}));
+ }
+
+ @Test
+ public void propagatePrivacyWithLineageFullReuseTest() {
+
propagationWithLineage(PrivacyConstraint.PrivacyLevel.PrivateAggregation);
+ }
+
+ private void propagationWithLineage(PrivacyConstraint.PrivacyLevel
privacyLevel) {
+
+ TestConfiguration config =
availableTestConfigurations.get("LineageReuse");
+ loadTestConfiguration(config);
+ fullDMLScriptName = SCRIPT_DIR + TEST_DIR +
config.getTestScript() + ".dml";
+
+ int n = 20;
+ int m = 20;
+ double[][] a = getRandomMatrix(m, n, -1, 1, 1, -1);
+ int k = 20;
+ double[][] b = getRandomMatrix(n, k, -1, 1, 1, -1);
+
+ PrivacyConstraint privacyConstraint = new
PrivacyConstraint(privacyLevel);
+ privacyConstraint.getFineGrainedPrivacy().put(new DataRange(new
long[]{0,0}, new long[]{4,4}), PrivacyConstraint.PrivacyLevel.Private);
+ MatrixCharacteristics aCharacteristics = new
MatrixCharacteristics(m, n, k, k);
+ MatrixCharacteristics bCharacteristics = new
MatrixCharacteristics(n, k, k, k);
+
+ writeInputMatrixWithMTD("A", a, false, aCharacteristics,
privacyConstraint);
+ writeInputMatrixWithMTD("B", b, false, bCharacteristics);
+
+ programArgs = new String[]{"-lineage", "reuse_full", "-nvargs",
+ "A=" + input("A"), "B=" + input("B"), "C=" +
output("C")};
+
+ runTest(true,false,null,-1);
+
+ finegrainedAssertions();
+
+ programArgs = new String[]{"-nvargs",
+ "A=" + input("A"), "B=" + input("B"), "C=" +
output("C")};
+ runTest(true,false,null,-1);
+
+ finegrainedAssertions();
+ }
+
+ private void finegrainedAssertions(){
+ String outputFineGrained =
readDMLMetaDataValueCatchException("C", OUTPUT_DIR,
DataExpression.FINE_GRAINED_PRIVACY);
+ Assert.assertEquals(
+
"{\"Private\":[[[0,0],[0,19]],[[1,0],[1,19]],[[2,0],[2,19]],[[3,0],[3,19]],[[4,0],[4,19]]],\"PrivateAggregation\":[]}",
+ outputFineGrained);
+ }
+}
diff --git a/src/test/scripts/functions/privacy/PrivacyLineageTest.dml
b/src/test/scripts/functions/privacy/PrivacyLineageTest.dml
new file mode 100644
index 0000000..d258cb9
--- /dev/null
+++ b/src/test/scripts/functions/privacy/PrivacyLineageTest.dml
@@ -0,0 +1,28 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+
+A = read($A)
+B = read($B)
+C = A %*% B;
+for (i in 1:10)
+ C = A %*% B;
+write(C, $C);
+
