tobiasrieger commented on a change in pull request #1075:
URL: https://github.com/apache/systemds/pull/1075#discussion_r499148662
##########
File path:
src/main/java/org/apache/sysds/runtime/instructions/cp/ListObject.java
##########
@@ -280,4 +301,124 @@ public String toString() {
sb.append(")");
return sb.toString();
}
+
+ /**
+ * Redirects the default java serialization via externalizable to our
default
+ * hadoop writable serialization for efficient broadcast/rdd
serialization.
+ *
+ * @param out object output
+ * @throws IOException if IOException occurs
+ */
+ @Override
+ public void writeExternal(ObjectOutput out) throws IOException {
+ // write out length
+ out.writeInt(getLength());
+ // write out num cacheable
+ out.writeInt(_nCacheable);
+
+ // write out names for named list
+ out.writeBoolean(getNames() != null);
+ if(getNames() != null) {
+ for (int i = 0; i < getLength(); i++) {
+ out.writeObject(_names.get(i));
+ }
+ }
+
+ // write out data
+ for(int i = 0; i < getLength(); i++) {
+ Data d = getData(i);
+ out.writeObject(d.getDataType());
+ out.writeObject(d.getValueType());
+ switch(d.getDataType()) {
+ case LIST:
+ ListObject lo = (ListObject) d;
+ out.writeObject(lo);
+ break;
+ case MATRIX:
+ MatrixObject mo = (MatrixObject) d;
+ MetaDataFormat md = (MetaDataFormat)
mo.getMetaData();
+ DataCharacteristics dc =
md.getDataCharacteristics();
+
+ out.writeObject(dc.getRows());
+ out.writeObject(dc.getCols());
+ out.writeObject(dc.getBlocksize());
+ out.writeObject(dc.getNonZeros());
+ out.writeObject(md.getFileFormat());
+
out.writeObject(mo.acquireReadAndRelease());
+ break;
+ case SCALAR:
+ ScalarObject so = (ScalarObject) d;
+ out.writeObject(so.getStringValue());
+ break;
+ default:
+ throw new DMLRuntimeException("Unable
to serialize datatype " + dataType);
+ }
+ }
+ }
+
+ /**
+ * Redirects the default java serialization via externalizable to our
default
+ * hadoop writable serialization for efficient broadcast/rdd
deserialization.
+ *
+ * @param in object input
+ * @throws IOException if IOException occurs
+ */
+ @Override
+ public void readExternal(ObjectInput in) throws IOException,
ClassNotFoundException {
+ // read in length
+ int length = in.readInt();
+ // read in num cacheable
+ _nCacheable = in.readInt();
+
+ // read in names
+ Boolean names = in.readBoolean();
+ if(names) {
+ _names = new ArrayList<>();
+ for (int i = 0; i < length; i++) {
+ _names.add((String) in.readObject());
+ }
+ }
+
+ // read in data
+ for(int i = 0; i < length; i++) {
+ DataType dataType = (DataType) in.readObject();
+ ValueType valueType = (ValueType) in.readObject();
+ Data d;
+ switch(dataType) {
+ case LIST:
+ d = (ListObject) in.readObject();
+ break;
+ case MATRIX:
+ long rows = (long) in.readObject();
+ long cols = (long) in.readObject();
+ int blockSize = (int) in.readObject();
+ long nonZeros = (long) in.readObject();
+ Types.FileFormat fileFormat =
(Types.FileFormat) in.readObject();
+
+ // construct objects and set meta data
+ MatrixCharacteristics
matrixCharacteristics = new MatrixCharacteristics(rows, cols, blockSize,
nonZeros);
+ MetaDataFormat metaDataFormat = new
MetaDataFormat(matrixCharacteristics, fileFormat);
+ MatrixBlock matrixBlock = (MatrixBlock)
in.readObject();
+
+ d = new MatrixObject(valueType,
Dag.getNextUniqueVarname(Types.DataType.MATRIX), metaDataFormat, matrixBlock);
Review comment:
I have realised, that all data objects have PCs, so I have just
generalised it. Take a look at the new commit 😊
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
