This is an automated email from the ASF dual-hosted git repository. mboehm7 pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/systemds.git
commit 58c6cf8cd76270a9b4d73043d89eb6077a313431 Author: Matthias Boehm <[email protected]> AuthorDate: Sat Sep 3 19:53:14 2022 +0200 [MINOR] Cleanup misc warnings and formatting issues --- .../compress/colgroup/ADictBasedColGroup.java | 1 + .../federated/monitoring/models/BaseModel.java | 1 + .../models/CoordinatorConnectionModel.java | 1 + .../monitoring/models/CoordinatorModel.java | 1 + .../monitoring/models/DataObjectModel.java | 2 +- .../federated/monitoring/models/EventModel.java | 2 +- .../monitoring/models/EventStageModel.java | 2 +- .../federated/monitoring/models/RequestModel.java | 2 +- .../monitoring/models/StatisticsModel.java | 1 + .../monitoring/models/StatisticsOptions.java | 1 + .../federated/monitoring/models/TrafficModel.java | 2 +- .../monitoring/models/UtilizationModel.java | 2 +- .../federated/monitoring/models/WorkerModel.java | 1 + .../paramserv/FederatedPSControlThread.java | 23 ++-- .../paramserv/homomorphicEncryption/PublicKey.java | 18 +-- .../homomorphicEncryption/SEALServer.java | 134 ++++++++++----------- .../ColGroupMorphingPerformanceCompare.java | 2 +- .../compress/colgroup/ColGroupNegativeTests.java | 7 ++ 18 files changed, 109 insertions(+), 94 deletions(-) diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ADictBasedColGroup.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ADictBasedColGroup.java index 0efdb4aa56..dc09fe9877 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ADictBasedColGroup.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ADictBasedColGroup.java @@ -35,6 +35,7 @@ import org.apache.sysds.runtime.data.SparseBlock; import org.apache.sysds.runtime.matrix.data.MatrixBlock; public abstract class ADictBasedColGroup extends AColGroupCompressed { + private static final long serialVersionUID = -3737025296618703668L; /** Distinct value tuples associated with individual bitmaps. */ protected ADictionary _dict; diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/BaseModel.java b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/BaseModel.java index dbd3320c01..3d1f5e9205 100644 --- a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/BaseModel.java +++ b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/BaseModel.java @@ -22,5 +22,6 @@ package org.apache.sysds.runtime.controlprogram.federated.monitoring.models; import java.io.Serializable; public abstract class BaseModel implements Serializable { + private static final long serialVersionUID = 5565981270528383999L; public Long id; } diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/CoordinatorConnectionModel.java b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/CoordinatorConnectionModel.java index dc28232225..08965ad1b5 100644 --- a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/CoordinatorConnectionModel.java +++ b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/CoordinatorConnectionModel.java @@ -20,6 +20,7 @@ package org.apache.sysds.runtime.controlprogram.federated.monitoring.models; public abstract class CoordinatorConnectionModel extends BaseModel { + private static final long serialVersionUID = 918360814223266197L; public Long coordinatorId; private String coordinatorHostId; private static final String localhostIp = "127.0.0.1"; diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/CoordinatorModel.java b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/CoordinatorModel.java index 98f25ee267..08a6938735 100644 --- a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/CoordinatorModel.java +++ b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/CoordinatorModel.java @@ -20,6 +20,7 @@ package org.apache.sysds.runtime.controlprogram.federated.monitoring.models; public class CoordinatorModel extends BaseModel { + private static final long serialVersionUID = 4116787631938152573L; public String name; public String host; public Long processId; diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/DataObjectModel.java b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/DataObjectModel.java index 5e20ba2887..c6c14e7390 100644 --- a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/DataObjectModel.java +++ b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/DataObjectModel.java @@ -20,7 +20,7 @@ package org.apache.sysds.runtime.controlprogram.federated.monitoring.models; public class DataObjectModel extends BaseModel { - + private static final long serialVersionUID = 7914784187151939188L; public Long workerId; public String varName; public String dataType; diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/EventModel.java b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/EventModel.java index fc4694c935..e3e4dc3398 100644 --- a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/EventModel.java +++ b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/EventModel.java @@ -24,7 +24,7 @@ import java.util.List; import java.util.stream.Collectors; public class EventModel extends CoordinatorConnectionModel { - + private static final long serialVersionUID = -5597621916956632690L; public Long workerId; private String coordinatorName; public List<EventStageModel> stages; diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/EventStageModel.java b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/EventStageModel.java index 1171a7c25c..ee4205a124 100644 --- a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/EventStageModel.java +++ b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/EventStageModel.java @@ -22,7 +22,7 @@ package org.apache.sysds.runtime.controlprogram.federated.monitoring.models; import java.time.LocalDateTime; public class EventStageModel extends BaseModel { - + private static final long serialVersionUID = -6867424341266726981L; public Long eventId; public String operation; public LocalDateTime startTime; diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/RequestModel.java b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/RequestModel.java index 81171e9311..ff971c129e 100644 --- a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/RequestModel.java +++ b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/RequestModel.java @@ -20,7 +20,7 @@ package org.apache.sysds.runtime.controlprogram.federated.monitoring.models; public class RequestModel extends CoordinatorConnectionModel { - + private static final long serialVersionUID = -5376072120202921000L; public Long workerId; public String type; public Long count; diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/StatisticsModel.java b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/StatisticsModel.java index cdbe6e73c5..613b286a27 100644 --- a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/StatisticsModel.java +++ b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/StatisticsModel.java @@ -23,6 +23,7 @@ import java.util.List; import java.util.stream.Collectors; public class StatisticsModel extends BaseModel { + private static final long serialVersionUID = -2492467768854934429L; public List<UtilizationModel> utilization; public List<TrafficModel> traffic; public List<EventModel> events; diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/StatisticsOptions.java b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/StatisticsOptions.java index bcbc57357d..cbbf4cd915 100644 --- a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/StatisticsOptions.java +++ b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/StatisticsOptions.java @@ -20,6 +20,7 @@ package org.apache.sysds.runtime.controlprogram.federated.monitoring.models; public class StatisticsOptions extends BaseModel { + private static final long serialVersionUID = 2524032122999491726L; public int rowCount = 20; public boolean utilization = true; public boolean traffic = true; diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/TrafficModel.java b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/TrafficModel.java index 9d0d1d160e..617324da64 100644 --- a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/TrafficModel.java +++ b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/TrafficModel.java @@ -22,7 +22,7 @@ package org.apache.sysds.runtime.controlprogram.federated.monitoring.models; import java.time.LocalDateTime; public class TrafficModel extends CoordinatorConnectionModel { - + private static final long serialVersionUID = 5042814368347405998L; public Long workerId; public LocalDateTime timestamp; public Long byteAmount; diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/UtilizationModel.java b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/UtilizationModel.java index d92555a3ee..b3b1076814 100644 --- a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/UtilizationModel.java +++ b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/UtilizationModel.java @@ -22,7 +22,7 @@ package org.apache.sysds.runtime.controlprogram.federated.monitoring.models; import java.time.LocalDateTime; public class UtilizationModel extends BaseModel { - + private static final long serialVersionUID = 6984053518916899551L; public Long workerId; public LocalDateTime timestamp; public double cpuUsage; diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/WorkerModel.java b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/WorkerModel.java index 922fe9239a..f1065b1bae 100644 --- a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/WorkerModel.java +++ b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/monitoring/models/WorkerModel.java @@ -20,6 +20,7 @@ package org.apache.sysds.runtime.controlprogram.federated.monitoring.models; public class WorkerModel extends BaseModel { + private static final long serialVersionUID = -7516912892237220796L; public String name; public String address; diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/paramserv/FederatedPSControlThread.java b/src/main/java/org/apache/sysds/runtime/controlprogram/paramserv/FederatedPSControlThread.java index 0c984698c1..09fd0a3349 100644 --- a/src/main/java/org/apache/sysds/runtime/controlprogram/paramserv/FederatedPSControlThread.java +++ b/src/main/java/org/apache/sysds/runtime/controlprogram/paramserv/FederatedPSControlThread.java @@ -293,14 +293,14 @@ public class FederatedPSControlThread extends PSWorker implements Callable<Void> byte[] _partial_pubkey_a; - protected SetupHEFederatedWorker(byte[] partial_pubkey_a) { - // delegate everything to parent class. set modelAvg to true, as it is the only supported case - super(); - _partial_pubkey_a = partial_pubkey_a; - } - - @Override - public FederatedResponse execute(ExecutionContext ec, Data... data) { + protected SetupHEFederatedWorker(byte[] partial_pubkey_a) { + // delegate everything to parent class. set modelAvg to true, as it is the only supported case + super(); + _partial_pubkey_a = partial_pubkey_a; + } + + @Override + public FederatedResponse execute(ExecutionContext ec, Data... data) { // TODO: set other CKKS parameters // TODO generate partial public key NativeHEHelper.initialize(); @@ -315,9 +315,10 @@ public class FederatedPSControlThread extends PSWorker implements Callable<Void> } return new FederatedResponse(FederatedResponse.ResponseType.SUCCESS, partial_pubkey); - } - } - /** + } + } + + /** * Teardown UDF executed on the federated worker */ private static class SetPublicKeyFederatedWorker extends FederatedUDF { diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/paramserv/homomorphicEncryption/PublicKey.java b/src/main/java/org/apache/sysds/runtime/controlprogram/paramserv/homomorphicEncryption/PublicKey.java index 96fd415308..bb3dea4db7 100644 --- a/src/main/java/org/apache/sysds/runtime/controlprogram/paramserv/homomorphicEncryption/PublicKey.java +++ b/src/main/java/org/apache/sysds/runtime/controlprogram/paramserv/homomorphicEncryption/PublicKey.java @@ -22,15 +22,15 @@ package org.apache.sysds.runtime.controlprogram.paramserv.homomorphicEncryption; import java.io.Serializable; public class PublicKey implements Serializable { - private static final long serialVersionUID = 91289081237980123L; + private static final long serialVersionUID = 91289081237980123L; - private final byte[] _data; + private final byte[] _data; - public PublicKey(byte[] data) { - _data = data; - } - - public byte[] getData() { - return _data; - } + public PublicKey(byte[] data) { + _data = data; + } + + public byte[] getData() { + return _data; + } } diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/paramserv/homomorphicEncryption/SEALServer.java b/src/main/java/org/apache/sysds/runtime/controlprogram/paramserv/homomorphicEncryption/SEALServer.java index d6265c7f6d..acf8509d91 100644 --- a/src/main/java/org/apache/sysds/runtime/controlprogram/paramserv/homomorphicEncryption/SEALServer.java +++ b/src/main/java/org/apache/sysds/runtime/controlprogram/paramserv/homomorphicEncryption/SEALServer.java @@ -35,78 +35,78 @@ import org.apache.sysds.runtime.meta.MetaDataFormat; import java.util.Arrays; public class SEALServer { - public SEALServer() { - // TODO take params here, like slot_count etc. - ctx = NativeHEHelper.initServer(); - } + public SEALServer() { + // TODO take params here, like slot_count etc. + ctx = NativeHEHelper.initServer(); + } - // this is a pointer to the context used by all native methods of this class - private final long ctx; - private byte[] _a; + // this is a pointer to the context used by all native methods of this class + private final long ctx; + private byte[] _a; - /** - * this generates the a constant. in a future version we want to generate this together with the clients to prevent misuse - * @return serialized a constant - */ - public synchronized byte[] generateA() { - if (_a == null) { - _a = NativeHEHelper.generateA(ctx); - } - return _a; - } + /** + * this generates the a constant. in a future version we want to generate this together with the clients to prevent misuse + * @return serialized a constant + */ + public synchronized byte[] generateA() { + if (_a == null) { + _a = NativeHEHelper.generateA(ctx); + } + return _a; + } - /** - * accumulates the given partial public keys into a public key, stores it in ctx and returns it - * @param partial_public_keys an array of partial public keys generated with SEALServer::generatePartialPublicKey - * @return the aggregated public key - */ - public PublicKey aggregatePartialPublicKeys(PublicKey[] partial_public_keys) { - return new PublicKey(NativeHEHelper.aggregatePartialPublicKeys(ctx, extractRawData(partial_public_keys))); - } + /** + * accumulates the given partial public keys into a public key, stores it in ctx and returns it + * @param partial_public_keys an array of partial public keys generated with SEALServer::generatePartialPublicKey + * @return the aggregated public key + */ + public PublicKey aggregatePartialPublicKeys(PublicKey[] partial_public_keys) { + return new PublicKey(NativeHEHelper.aggregatePartialPublicKeys(ctx, extractRawData(partial_public_keys))); + } - /** - * accumulates the given ciphertext blocks into a sum ciphertext and returns it - * @param ciphertexts ciphertexts encrypted with the partial public keys - * @return the accumulated ciphertext (which is the homomorphic sum of ciphertexts) - */ - public CiphertextMatrix accumulateCiphertexts(CiphertextMatrix[] ciphertexts) { - return new CiphertextMatrix(ciphertexts[0].getDims(), ciphertexts[0].getDataCharacteristics(), NativeHEHelper.accumulateCiphertexts(ctx, extractRawData(ciphertexts))); - } + /** + * accumulates the given ciphertext blocks into a sum ciphertext and returns it + * @param ciphertexts ciphertexts encrypted with the partial public keys + * @return the accumulated ciphertext (which is the homomorphic sum of ciphertexts) + */ + public CiphertextMatrix accumulateCiphertexts(CiphertextMatrix[] ciphertexts) { + return new CiphertextMatrix(ciphertexts[0].getDims(), ciphertexts[0].getDataCharacteristics(), NativeHEHelper.accumulateCiphertexts(ctx, extractRawData(ciphertexts))); + } - /** - * averages the partial decryptions - * @param encrypted_sum is the result of accumulateCiphertexts() - * @param partial_plaintexts is the result of SEALServer::partiallyDecrypt of each ciphertext fed into accumulateCiphertexts - * @return the unencrypted, element-wise average of the original matrices - */ - public MatrixObject average(CiphertextMatrix encrypted_sum, PlaintextMatrix[] partial_plaintexts) { - double[] raw_result = NativeHEHelper.average(ctx, encrypted_sum.getData(), extractRawData(partial_plaintexts)); - int[] dims = encrypted_sum.getDims(); - int result_len = Arrays.stream(dims).reduce(1, (x,y) -> x*y); - DataCharacteristics dc = encrypted_sum.getDataCharacteristics(); + /** + * averages the partial decryptions + * @param encrypted_sum is the result of accumulateCiphertexts() + * @param partial_plaintexts is the result of SEALServer::partiallyDecrypt of each ciphertext fed into accumulateCiphertexts + * @return the unencrypted, element-wise average of the original matrices + */ + public MatrixObject average(CiphertextMatrix encrypted_sum, PlaintextMatrix[] partial_plaintexts) { + double[] raw_result = NativeHEHelper.average(ctx, encrypted_sum.getData(), extractRawData(partial_plaintexts)); + int[] dims = encrypted_sum.getDims(); + int result_len = Arrays.stream(dims).reduce(1, (x,y) -> x*y); + DataCharacteristics dc = encrypted_sum.getDataCharacteristics(); - DenseBlock new_dense_block = DenseBlockFactory.createDenseBlock(Arrays.copyOf(raw_result, result_len), dims); - MatrixBlock new_matrix_block = new MatrixBlock((int)dc.getRows(), (int)dc.getCols(), new_dense_block); - MatrixObject new_mo = new MatrixObject(Types.ValueType.FP64, OptimizerUtils.getUniqueTempFileName(), new MetaDataFormat(dc, Types.FileFormat.BINARY)); - new_mo.acquireModify(new_matrix_block); - new_mo.release(); - return new_mo; - } + DenseBlock new_dense_block = DenseBlockFactory.createDenseBlock(Arrays.copyOf(raw_result, result_len), dims); + MatrixBlock new_matrix_block = new MatrixBlock((int)dc.getRows(), (int)dc.getCols(), new_dense_block); + MatrixObject new_mo = new MatrixObject(Types.ValueType.FP64, OptimizerUtils.getUniqueTempFileName(), new MetaDataFormat(dc, Types.FileFormat.BINARY)); + new_mo.acquireModify(new_matrix_block); + new_mo.release(); + return new_mo; + } - private static byte[][] extractRawData(Encrypted[] data) { - byte[][] raw_data = new byte[data.length][]; - for (int i = 0; i < data.length; i++) { - raw_data[i] = data[i].getData(); - } - return raw_data; - } + private static byte[][] extractRawData(Encrypted[] data) { + byte[][] raw_data = new byte[data.length][]; + for (int i = 0; i < data.length; i++) { + raw_data[i] = data[i].getData(); + } + return raw_data; + } - // TODO: extract an interface for this and use it here - private static byte[][] extractRawData(PublicKey[] data) { - byte[][] raw_data = new byte[data.length][]; - for (int i = 0; i < data.length; i++) { - raw_data[i] = data[i].getData(); - } - return raw_data; - } -} \ No newline at end of file + // TODO: extract an interface for this and use it here + private static byte[][] extractRawData(PublicKey[] data) { + byte[][] raw_data = new byte[data.length][]; + for (int i = 0; i < data.length; i++) { + raw_data[i] = data[i].getData(); + } + return raw_data; + } +} diff --git a/src/test/java/org/apache/sysds/test/component/compress/colgroup/ColGroupMorphingPerformanceCompare.java b/src/test/java/org/apache/sysds/test/component/compress/colgroup/ColGroupMorphingPerformanceCompare.java index ca15628c64..14962b038a 100644 --- a/src/test/java/org/apache/sysds/test/component/compress/colgroup/ColGroupMorphingPerformanceCompare.java +++ b/src/test/java/org/apache/sysds/test/component/compress/colgroup/ColGroupMorphingPerformanceCompare.java @@ -150,7 +150,7 @@ public class ColGroupMorphingPerformanceCompare { } protected static class SDCNoMorph extends ColGroupSDC { - + private static final long serialVersionUID = -7157464508602251065L; private final MatrixBlock mbDict; protected SDCNoMorph(int numRows) { diff --git a/src/test/java/org/apache/sysds/test/component/compress/colgroup/ColGroupNegativeTests.java b/src/test/java/org/apache/sysds/test/component/compress/colgroup/ColGroupNegativeTests.java index fb94a9a8c8..485584e4fa 100644 --- a/src/test/java/org/apache/sysds/test/component/compress/colgroup/ColGroupNegativeTests.java +++ b/src/test/java/org/apache/sysds/test/component/compress/colgroup/ColGroupNegativeTests.java @@ -138,6 +138,8 @@ public class ColGroupNegativeTests { } private class FakeIndexing extends IndexFunction { + private static final long serialVersionUID = -4099420257856761251L; + protected FakeIndexing() { } @@ -173,12 +175,16 @@ public class ColGroupNegativeTests { } private class FakeValueFunction extends ValueFunction { + private static final long serialVersionUID = -585186573175954738L; + private FakeValueFunction() { } } private class FakeAPreAgg extends APreAgg { + private static final long serialVersionUID = 8759470530917794282L; + private FakeAPreAgg() { super(new int[1], Dictionary.createNoCheck(new double[13]), null); } @@ -339,6 +345,7 @@ public class ColGroupNegativeTests { } private class FakeDictBasedColGroup extends ADictBasedColGroup { + private static final long serialVersionUID = 7578204757649117273L; private FakeDictBasedColGroup() { super(null, null);
