Repository: systemml Updated Branches: refs/heads/master 32620f294 -> a4962f39a
[SYSTEMML-1928] Removed Tensorboard visualization code from Caffe2DML - This avoids compilation error on powerpc. Closes #671. Project: http://git-wip-us.apache.org/repos/asf/systemml/repo Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/a4962f39 Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/a4962f39 Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/a4962f39 Branch: refs/heads/master Commit: a4962f39a065bce89dd1c45635e318b9440e5491 Parents: 32620f2 Author: Niketan Pansare <[email protected]> Authored: Thu Sep 21 13:37:49 2017 -0700 Committer: Niketan Pansare <[email protected]> Committed: Thu Sep 21 13:37:49 2017 -0700 ---------------------------------------------------------------------- docs/beginners-guide-caffe2dml.md | 2 +- pom.xml | 23 +- src/assembly/extra.xml | 2 - src/assembly/extra/LICENSE | 210 ------------------- .../udf/lib/Caffe2DMLVisualizeWrapper.java | 66 ------ .../apache/sysml/utils/TensorboardLogger.java | 177 ---------------- src/main/proto/tensorflow/event.proto | 82 -------- src/main/proto/tensorflow/summary.proto | 103 --------- src/main/python/LICENSE | 210 ------------------- src/main/python/systemml/mllearn/estimators.py | 33 +-- .../org/apache/sysml/api/dl/Caffe2DML.scala | 8 - .../org/apache/sysml/api/dl/DMLGenerator.scala | 89 +------- 12 files changed, 5 insertions(+), 1000 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/systemml/blob/a4962f39/docs/beginners-guide-caffe2dml.md ---------------------------------------------------------------------- diff --git a/docs/beginners-guide-caffe2dml.md b/docs/beginners-guide-caffe2dml.md index 12b21d4..220e02c 100644 --- a/docs/beginners-guide-caffe2dml.md +++ b/docs/beginners-guide-caffe2dml.md @@ -231,7 +231,7 @@ Dml.g4 ---> antlr ---> DmlLexer.java, DmlListener.java, DmlParser.java --- caffe.proto ---> protoc ---> target/generated-sources/caffe/Caffe.java ---> parse caffe_network.proto, caffe_solver.proto ``` -Again, the SystemML engine doesnot invoke (or depend on) Caffe and TensorFlow for any of its runtime operators. +Again, the SystemML engine doesnot invoke (or depend on) Caffe for any of its runtime operators. Since the grammar files for the respective APIs (i.e. `caffe.proto`) are used by SystemML, we include their licenses in our jar files. http://git-wip-us.apache.org/repos/asf/systemml/blob/a4962f39/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index 7e04af5..c19e11c 100644 --- a/pom.xml +++ b/pom.xml @@ -270,9 +270,6 @@ </archive> <excludes> <exclude>**/caffe/*</exclude> - <exclude>**/org/tensorflow/*</exclude> - <exclude>**/org/tensorflow/framework/*</exclude> - <exclude>**/org/tensorflow/util/*</exclude> <exclude>**/org/apache/sysml/api/dl/*</exclude> </excludes> </configuration> @@ -366,22 +363,6 @@ </outputDirectories> </configuration> </execution> - <execution> - <id>tf-sources</id> - <phase>generate-sources</phase> - <goals> - <goal>run</goal> - </goals> - <configuration> - <protocVersion>3.0.0</protocVersion> <!-- 2.4.1, 2.5.0, 2.6.1, 3.0.0 --> - <inputDirectories> - <include>src/main/proto/tensorflow</include> - </inputDirectories> - <outputDirectories> - <include>src/main/java</include> - </outputDirectories> - </configuration> - </execution> </executions> </plugin> @@ -884,8 +865,6 @@ <exclude>src/main/cpp/kernels/SystemML.ptx</exclude> <!-- Proto files --> <exclude>src/main/proto/caffe/caffe.proto</exclude> - <exclude>src/main/proto/tensorflow/event.proto</exclude> - <exclude>src/main/proto/tensorflow/summary.proto</exclude> <exclude>scripts/nn/examples/caffe2dml/models/**/*</exclude> <!-- Test Validation files --> <exclude>src/test/scripts/functions/jmlc/**/*.impute</exclude> @@ -1045,7 +1024,7 @@ org.apache.sysml.runtime.matrix.data (for MatrixIndexes, MatrixBlock, etc) org.apache.sysml.udf --> - <excludePackageNames>caffe:org.tensorflow:org.apache.sysml.conf:org.apache.sysml.debug:org.apache.sysml.hops:org.apache.sysml.lops:org.apache.sysml.parser:org.apache.sysml.runtime.controlprogram:org.apache.sysml.runtime.functionobjects:org.apache.sysml.runtime.instructions.cp:org.apache.sysml.runtime.instructions.cpfile:org.apache.sysml.runtime.instructions.mr:org.apache.sysml.runtime.instructions.spark.data:org.apache.sysml.runtime.instructions.spark.functions:org.apache.sysml.runtime.io:org.apache.sysml.runtime.matrix.data.hadoopfix:org.apache.sysml.runtime.matrix.mapred:org.apache.sysml.runtime.matrix.operators:org.apache.sysml.runtime.matrix.sort:org.apache.sysml.runtime.transform:org.apache.sysml.runtime.util:org.apache.sysml.utils:org.apache.sysml.yarn</excludePackageNames> + <excludePackageNames>caffe:org.apache.sysml.conf:org.apache.sysml.debug:org.apache.sysml.hops:org.apache.sysml.lops:org.apache.sysml.parser:org.apache.sysml.runtime.controlprogram:org.apache.sysml.runtime.functionobjects:org.apache.sysml.runtime.instructions.cp:org.apache.sysml.runtime.instructions.cpfile:org.apache.sysml.runtime.instructions.mr:org.apache.sysml.runtime.instructions.spark.data:org.apache.sysml.runtime.instructions.spark.functions:org.apache.sysml.runtime.io:org.apache.sysml.runtime.matrix.data.hadoopfix:org.apache.sysml.runtime.matrix.mapred:org.apache.sysml.runtime.matrix.operators:org.apache.sysml.runtime.matrix.sort:org.apache.sysml.runtime.transform:org.apache.sysml.runtime.util:org.apache.sysml.utils:org.apache.sysml.yarn</excludePackageNames> <additionalparam>${javadoc.opts}</additionalparam> </configuration> <executions> http://git-wip-us.apache.org/repos/asf/systemml/blob/a4962f39/src/assembly/extra.xml ---------------------------------------------------------------------- diff --git a/src/assembly/extra.xml b/src/assembly/extra.xml index c781615..9495b86 100644 --- a/src/assembly/extra.xml +++ b/src/assembly/extra.xml @@ -43,8 +43,6 @@ <directory>${basedir}/target/classes</directory> <includes> <include>**/caffe/*</include> - <include>**/org/tensorflow/framework/*</include> - <include>**/org/tensorflow/util/*</include> <include>**/org/apache/sysml/api/dl/*</include> </includes> <outputDirectory>.</outputDirectory> http://git-wip-us.apache.org/repos/asf/systemml/blob/a4962f39/src/assembly/extra/LICENSE ---------------------------------------------------------------------- diff --git a/src/assembly/extra/LICENSE b/src/assembly/extra/LICENSE index c495849..e08b593 100644 --- a/src/assembly/extra/LICENSE +++ b/src/assembly/extra/LICENSE @@ -253,216 +253,6 @@ license and copyright terms herein. =============================================================================== -The proto files (src/main/proto/tensorflow/event.proto and src/main/proto/tensorflow/summary.proto) is part of TensorFlow project, -which is used to generate org.tensorflow package. -TensorFlow are distributed under the Apache License 2.0. - -Copyright 2017 The TensorFlow Authors. All rights reserved. - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright 2017, The TensorFlow Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - -=============================================================================== - The following compile-scope dependencies come under the MIT License JCuda (jcuda.org) http://git-wip-us.apache.org/repos/asf/systemml/blob/a4962f39/src/main/java/org/apache/sysml/udf/lib/Caffe2DMLVisualizeWrapper.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/udf/lib/Caffe2DMLVisualizeWrapper.java b/src/main/java/org/apache/sysml/udf/lib/Caffe2DMLVisualizeWrapper.java deleted file mode 100644 index 15c867b..0000000 --- a/src/main/java/org/apache/sysml/udf/lib/Caffe2DMLVisualizeWrapper.java +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.sysml.udf.lib; - -import org.apache.sysml.udf.FunctionParameter; -import org.apache.sysml.udf.PackageFunction; -import org.apache.sysml.udf.Scalar; -import org.apache.sysml.udf.Scalar.ScalarValueType; -import org.apache.sysml.utils.TensorboardLogger; - -public class Caffe2DMLVisualizeWrapper extends PackageFunction -{ - private static final long serialVersionUID = 1L; - private Scalar _ret; - - @Override - public int getNumFunctionOutputs() { - return 1; - } - - @Override - public FunctionParameter getFunctionOutput(int pos) { - if (pos == 0) - return _ret; - - throw new RuntimeException( - "Invalid function output being requested"); - } - - @Override - public void execute() { - String layerName = ((Scalar) this.getFunctionInput(0)).getValue(); - String varType = ((Scalar) this.getFunctionInput(1)).getValue(); - String aggFn = ((Scalar) this.getFunctionInput(2)).getValue(); - double x = Double.parseDouble(((Scalar) this.getFunctionInput(3)).getValue()); - double y = Double.parseDouble(((Scalar) this.getFunctionInput(4)).getValue()); - String logDir = ((Scalar) this.getFunctionInput(5)).getValue(); - - String key = null; - if(aggFn.equals("training_loss") || aggFn.equals("validation_loss") || - aggFn.equals("training_accuracy") || aggFn.equals("validation_accuracy")) - key = aggFn; - else - key = aggFn + "_" + varType + "_" + layerName; - TensorboardLogger.writeScalar(logDir, key, (long)x, (float)y); - _ret = new Scalar(ScalarValueType.Double, String.valueOf(1)); - } - -} http://git-wip-us.apache.org/repos/asf/systemml/blob/a4962f39/src/main/java/org/apache/sysml/utils/TensorboardLogger.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/utils/TensorboardLogger.java b/src/main/java/org/apache/sysml/utils/TensorboardLogger.java deleted file mode 100644 index 245d757..0000000 --- a/src/main/java/org/apache/sysml/utils/TensorboardLogger.java +++ /dev/null @@ -1,177 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.sysml.utils; - -import java.io.File; -import java.io.FileOutputStream; -import java.io.IOException; -import java.util.zip.Checksum; -import org.tensorflow.framework.Summary; -import org.tensorflow.util.Event; - -import com.google.common.primitives.Ints; -import com.google.common.primitives.Longs; - -public class TensorboardLogger { - private static Crc32c crc32 = new Crc32c(); - - /** - * Writes scalar of given value in tensorboard format - * - * @param logDir log directory of tensorboard - * @param tag scalar tag (for example: training_loss, validation_loss, ...) - * @param step usually the iteration number - * @param value value of the scalar - */ - public static void writeScalar(String logDir, String tag, long step, float value) { - String filePath = logDir + File.separator + "tfevents.event_systemml_scalar"; - try { - FileOutputStream outputStream = new FileOutputStream(filePath, true); - Event event = Event.newBuilder() - .setWallTime(System.currentTimeMillis() / 1e3) - .setStep(step) - .setSummary(Summary.newBuilder().addValue( - Summary.Value.newBuilder().setTag(tag).setSimpleValue(value) - ).build()) - .build(); - byte[] eventString = event.toByteArray(); - byte[] header = reverse(Longs.toByteArray((long)eventString.length)); - write(outputStream, header); - write(outputStream, eventString); - outputStream.close(); - } - catch(IOException e) { - throw new RuntimeException("Error writing event in tensorboard directory:" + filePath, e); - } - } - - private static void write(FileOutputStream outputStream, byte[] byteString) throws IOException { - outputStream.write(byteString); - outputStream.write(reverse(Ints.toByteArray((int)maskedCRC32(byteString)))); - } - - private static byte[] reverse(byte[] nums) { - byte[] reversed = new byte[nums.length]; - for (int i=0; i<nums.length; i++) { - reversed[i] = nums[nums.length - 1 - i]; - } - return reversed; - } - - private static long maskedCRC32(byte[] data){ - crc32.reset(); - crc32.update(data, 0, data.length); - long x = u32(crc32.getValue()); - return u32(((x >> 15) | u32(x << 17)) + 0xa282ead8); - } - - private static long u32(long x){ - return x & 0xffffffff; - } -} - -class Crc32c implements Checksum { - private static final int[] crcTable = { - 0x00000000, 0xF26B8303, 0xE13B70F7, 0x1350F3F4, - 0xC79A971F, 0x35F1141C, 0x26A1E7E8, 0xD4CA64EB, - 0x8AD958CF, 0x78B2DBCC, 0x6BE22838, 0x9989AB3B, - 0x4D43CFD0, 0xBF284CD3, 0xAC78BF27, 0x5E133C24, - 0x105EC76F, 0xE235446C, 0xF165B798, 0x030E349B, - 0xD7C45070, 0x25AFD373, 0x36FF2087, 0xC494A384, - 0x9A879FA0, 0x68EC1CA3, 0x7BBCEF57, 0x89D76C54, - 0x5D1D08BF, 0xAF768BBC, 0xBC267848, 0x4E4DFB4B, - 0x20BD8EDE, 0xD2D60DDD, 0xC186FE29, 0x33ED7D2A, - 0xE72719C1, 0x154C9AC2, 0x061C6936, 0xF477EA35, - 0xAA64D611, 0x580F5512, 0x4B5FA6E6, 0xB93425E5, - 0x6DFE410E, 0x9F95C20D, 0x8CC531F9, 0x7EAEB2FA, - 0x30E349B1, 0xC288CAB2, 0xD1D83946, 0x23B3BA45, - 0xF779DEAE, 0x05125DAD, 0x1642AE59, 0xE4292D5A, - 0xBA3A117E, 0x4851927D, 0x5B016189, 0xA96AE28A, - 0x7DA08661, 0x8FCB0562, 0x9C9BF696, 0x6EF07595, - 0x417B1DBC, 0xB3109EBF, 0xA0406D4B, 0x522BEE48, - 0x86E18AA3, 0x748A09A0, 0x67DAFA54, 0x95B17957, - 0xCBA24573, 0x39C9C670, 0x2A993584, 0xD8F2B687, - 0x0C38D26C, 0xFE53516F, 0xED03A29B, 0x1F682198, - 0x5125DAD3, 0xA34E59D0, 0xB01EAA24, 0x42752927, - 0x96BF4DCC, 0x64D4CECF, 0x77843D3B, 0x85EFBE38, - 0xDBFC821C, 0x2997011F, 0x3AC7F2EB, 0xC8AC71E8, - 0x1C661503, 0xEE0D9600, 0xFD5D65F4, 0x0F36E6F7, - 0x61C69362, 0x93AD1061, 0x80FDE395, 0x72966096, - 0xA65C047D, 0x5437877E, 0x4767748A, 0xB50CF789, - 0xEB1FCBAD, 0x197448AE, 0x0A24BB5A, 0xF84F3859, - 0x2C855CB2, 0xDEEEDFB1, 0xCDBE2C45, 0x3FD5AF46, - 0x7198540D, 0x83F3D70E, 0x90A324FA, 0x62C8A7F9, - 0xB602C312, 0x44694011, 0x5739B3E5, 0xA55230E6, - 0xFB410CC2, 0x092A8FC1, 0x1A7A7C35, 0xE811FF36, - 0x3CDB9BDD, 0xCEB018DE, 0xDDE0EB2A, 0x2F8B6829, - 0x82F63B78, 0x709DB87B, 0x63CD4B8F, 0x91A6C88C, - 0x456CAC67, 0xB7072F64, 0xA457DC90, 0x563C5F93, - 0x082F63B7, 0xFA44E0B4, 0xE9141340, 0x1B7F9043, - 0xCFB5F4A8, 0x3DDE77AB, 0x2E8E845F, 0xDCE5075C, - 0x92A8FC17, 0x60C37F14, 0x73938CE0, 0x81F80FE3, - 0x55326B08, 0xA759E80B, 0xB4091BFF, 0x466298FC, - 0x1871A4D8, 0xEA1A27DB, 0xF94AD42F, 0x0B21572C, - 0xDFEB33C7, 0x2D80B0C4, 0x3ED04330, 0xCCBBC033, - 0xA24BB5A6, 0x502036A5, 0x4370C551, 0xB11B4652, - 0x65D122B9, 0x97BAA1BA, 0x84EA524E, 0x7681D14D, - 0x2892ED69, 0xDAF96E6A, 0xC9A99D9E, 0x3BC21E9D, - 0xEF087A76, 0x1D63F975, 0x0E330A81, 0xFC588982, - 0xB21572C9, 0x407EF1CA, 0x532E023E, 0xA145813D, - 0x758FE5D6, 0x87E466D5, 0x94B49521, 0x66DF1622, - 0x38CC2A06, 0xCAA7A905, 0xD9F75AF1, 0x2B9CD9F2, - 0xFF56BD19, 0x0D3D3E1A, 0x1E6DCDEE, 0xEC064EED, - 0xC38D26C4, 0x31E6A5C7, 0x22B65633, 0xD0DDD530, - 0x0417B1DB, 0xF67C32D8, 0xE52CC12C, 0x1747422F, - 0x49547E0B, 0xBB3FFD08, 0xA86F0EFC, 0x5A048DFF, - 0x8ECEE914, 0x7CA56A17, 0x6FF599E3, 0x9D9E1AE0, - 0xD3D3E1AB, 0x21B862A8, 0x32E8915C, 0xC083125F, - 0x144976B4, 0xE622F5B7, 0xF5720643, 0x07198540, - 0x590AB964, 0xAB613A67, 0xB831C993, 0x4A5A4A90, - 0x9E902E7B, 0x6CFBAD78, 0x7FAB5E8C, 0x8DC0DD8F, - 0xE330A81A, 0x115B2B19, 0x020BD8ED, 0xF0605BEE, - 0x24AA3F05, 0xD6C1BC06, 0xC5914FF2, 0x37FACCF1, - 0x69E9F0D5, 0x9B8273D6, 0x88D28022, 0x7AB90321, - 0xAE7367CA, 0x5C18E4C9, 0x4F48173D, 0xBD23943E, - 0xF36E6F75, 0x0105EC76, 0x12551F82, 0xE03E9C81, - 0x34F4F86A, 0xC69F7B69, 0xD5CF889D, 0x27A40B9E, - 0x79B737BA, 0x8BDCB4B9, 0x988C474D, 0x6AE7C44E, - 0xBE2DA0A5, 0x4C4623A6, 0x5F16D052, 0xAD7D5351, - }; - - private int crc = ~0; - - public void update(byte[] buffer, int offset, int length) { - for (int i = offset; i < offset + length; i++) { - crc = crc32c(crc, buffer[i]); - } - } - public long getValue() { - return (crc ^ 0xFFFFFFFFL) & 0xFFFFFFFFL; - } - public void reset() { - crc = ~0; - } - private static int crc32c(int crc, int b) { - return crc >>> 8 ^ crcTable[(crc ^ b & 0xFF) & 0xFF]; - } - public void update(int arg0) { - throw new RuntimeException("Not implemented"); - } -} http://git-wip-us.apache.org/repos/asf/systemml/blob/a4962f39/src/main/proto/tensorflow/event.proto ---------------------------------------------------------------------- diff --git a/src/main/proto/tensorflow/event.proto b/src/main/proto/tensorflow/event.proto deleted file mode 100644 index c1816ff..0000000 --- a/src/main/proto/tensorflow/event.proto +++ /dev/null @@ -1,82 +0,0 @@ -syntax = "proto3"; - -package tensorflow; -option cc_enable_arenas = true; -option java_outer_classname = "EventProtos"; -option java_multiple_files = true; -option java_package = "org.tensorflow.util"; - -import "summary.proto"; - -// Protocol buffer representing an event that happened during -// the execution of a Brain model. -message Event { - // Timestamp of the event. - double wall_time = 1; - - // Global step of the event. - int64 step = 2; - - oneof what { - // An event file was started, with the specified version. - // This is use to identify the contents of the record IO files - // easily. Current version is "brain.Event:2". All versions - // start with "brain.Event:". - string file_version = 3; - // An encoded version of a GraphDef. - bytes graph_def = 4; - // A summary was generated. - Summary summary = 5; - // The user output a log message. Not all messages are logged, only ones - // generated via the Python tensorboard_logging module. - LogMessage log_message = 6; - // The state of the session which can be used for restarting after crashes. - SessionLog session_log = 7; - // The metadata returned by running a session.run() call. - TaggedRunMetadata tagged_run_metadata = 8; - // An encoded version of a MetaGraphDef. - bytes meta_graph_def = 9; - } -} - -// Protocol buffer used for logging messages to the events file. -message LogMessage { - enum Level { - UNKNOWN = 0; - // Note: The logging level 10 cannot be named DEBUG. Some software - // projects compile their C/C++ code with -DDEBUG in debug builds. So the - // C++ code generated from this file should not have an identifier named - // DEBUG. - DEBUGGING = 10; - INFO = 20; - WARN = 30; - ERROR = 40; - FATAL = 50; - } - Level level = 1; - string message = 2; -} - -// Protocol buffer used for logging session state. -message SessionLog { - enum SessionStatus { - STATUS_UNSPECIFIED = 0; - START = 1; - STOP = 2; - CHECKPOINT = 3; - } - - SessionStatus status = 1; - // This checkpoint_path contains both the path and filename. - string checkpoint_path = 2; - string msg = 3; -} - -// For logging the metadata output for a single session.run() call. -message TaggedRunMetadata { - // Tag name associated with this metadata. - string tag = 1; - // Byte-encoded version of the `RunMetadata` proto in order to allow lazy - // deserialization. - bytes run_metadata = 2; -} http://git-wip-us.apache.org/repos/asf/systemml/blob/a4962f39/src/main/proto/tensorflow/summary.proto ---------------------------------------------------------------------- diff --git a/src/main/proto/tensorflow/summary.proto b/src/main/proto/tensorflow/summary.proto deleted file mode 100644 index 29011fd..0000000 --- a/src/main/proto/tensorflow/summary.proto +++ /dev/null @@ -1,103 +0,0 @@ -syntax = "proto3"; - -package tensorflow; -option cc_enable_arenas = true; -option java_outer_classname = "SummaryProtos"; -option java_multiple_files = true; -option java_package = "org.tensorflow.framework"; - -// import "tensorflow/core/framework/tensor.proto"; - -// Metadata associated with a series of Summary data -message SummaryDescription { - // Hint on how plugins should process the data in this series. - // Supported values include "scalar", "histogram", "image", "audio" - string type_hint = 1; -} - -// Serialization format for histogram module in -// core/lib/histogram/histogram.h -message HistogramProto { - double min = 1; - double max = 2; - double num = 3; - double sum = 4; - double sum_squares = 5; - - // Parallel arrays encoding the bucket boundaries and the bucket values. - // bucket(i) is the count for the bucket i. The range for - // a bucket is: - // i == 0: -DBL_MAX .. bucket_limit(0) - // i != 0: bucket_limit(i-1) .. bucket_limit(i) - repeated double bucket_limit = 6 [packed = true]; - repeated double bucket = 7 [packed = true]; -}; - -// A Summary is a set of named values to be displayed by the -// visualizer. -// -// Summaries are produced regularly during training, as controlled by -// the "summary_interval_secs" attribute of the training operation. -// Summaries are also produced at the end of an evaluation. -message Summary { - message Image { - // Dimensions of the image. - int32 height = 1; - int32 width = 2; - // Valid colorspace values are - // 1 - grayscale - // 2 - grayscale + alpha - // 3 - RGB - // 4 - RGBA - // 5 - DIGITAL_YUV - // 6 - BGRA - int32 colorspace = 3; - // Image data in encoded format. All image formats supported by - // image_codec::CoderUtil can be stored here. - bytes encoded_image_string = 4; - } - - message Audio { - // Sample rate of the audio in Hz. - float sample_rate = 1; - // Number of channels of audio. - int64 num_channels = 2; - // Length of the audio in frames (samples per channel). - int64 length_frames = 3; - // Encoded audio data and its associated RFC 2045 content type (e.g. - // "audio/wav"). - bytes encoded_audio_string = 4; - string content_type = 5; - } - - message Value { - // Name of the node that output this summary; in general, the name of a - // TensorSummary node. If the node in question has multiple outputs, then - // a ":\d+" suffix will be appended, like "some_op:13". - // Might not be set for legacy summaries (i.e. those not using the tensor - // value field) - string node_name = 7; - - // Tag name for the data. Will only be used by legacy summaries - // (ie. those not using the tensor value field) - // For legacy summaries, will be used as the title of the graph - // in the visualizer. - // - // Tag is usually "op_name:value_name", where "op_name" itself can have - // structure to indicate grouping. - string tag = 1; - - // Value associated with the tag. - oneof value { - float simple_value = 2; - bytes obsolete_old_style_histogram = 3; - Image image = 4; - HistogramProto histo = 5; - Audio audio = 6; - // TensorProto tensor = 8; - } - } - - // Set of values for the summary. - repeated Value value = 1; -} http://git-wip-us.apache.org/repos/asf/systemml/blob/a4962f39/src/main/python/LICENSE ---------------------------------------------------------------------- diff --git a/src/main/python/LICENSE b/src/main/python/LICENSE index 02d679c..ed7417c 100644 --- a/src/main/python/LICENSE +++ b/src/main/python/LICENSE @@ -296,213 +296,3 @@ CONTRIBUTION AGREEMENT By contributing to the BVLC/caffe repository through pull-request, comment, or otherwise, the contributor releases their content to the license and copyright terms herein. - -=============================================================================== - -The proto files (src/main/proto/tensorflow/event.proto and src/main/proto/tensorflow/summary.proto) is part of TensorFlow project, -which is used to generate org.tensorflow package. -TensorFlow are distributed under the Apache License 2.0. - -Copyright 2017 The TensorFlow Authors. All rights reserved. - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright 2017, The TensorFlow Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. \ No newline at end of file http://git-wip-us.apache.org/repos/asf/systemml/blob/a4962f39/src/main/python/systemml/mllearn/estimators.py ---------------------------------------------------------------------- diff --git a/src/main/python/systemml/mllearn/estimators.py b/src/main/python/systemml/mllearn/estimators.py index 1df1bb8..50412aa 100644 --- a/src/main/python/systemml/mllearn/estimators.py +++ b/src/main/python/systemml/mllearn/estimators.py @@ -45,7 +45,6 @@ def assemble(sparkSession, pdf, inputCols, outputCol): class BaseSystemMLEstimator(Estimator): features_col = 'features' label_col = 'label' - do_visualize = False def set_features_col(self, colName): """ @@ -756,7 +755,7 @@ class Caffe2DML(BaseSystemMLClassifier): >>> caffe2DML = Caffe2DML(spark, 'lenet_solver.proto').set(max_iter=500) >>> caffe2DML.fit(X, y) """ - def __init__(self, sparkSession, solver, input_shape, transferUsingDF=False, tensorboard_log_dir=None): + def __init__(self, sparkSession, solver, input_shape, transferUsingDF=False): """ Performs training/prediction for a given caffe network. @@ -766,7 +765,6 @@ class Caffe2DML(BaseSystemMLClassifier): solver: caffe solver file path input_shape: 3-element list (number of channels, input height, input width) transferUsingDF: whether to pass the input dataset via PySpark DataFrame (default: False) - tensorboard_log_dir: directory to store the event logs (default: None, we use a temporary directory) """ self.sparkSession = sparkSession self.sc = sparkSession._sc @@ -779,9 +777,6 @@ class Caffe2DML(BaseSystemMLClassifier): self.estimator = self.sc._jvm.org.apache.sysml.api.dl.Caffe2DML(self.sc._jsc.sc(), solver, str(input_shape[0]), str(input_shape[1]), str(input_shape[2])) self.transferUsingDF = transferUsingDF self.setOutputRawPredictionsToFalse = False - self.visualize_called = False - if tensorboard_log_dir is not None: - self.estimator.setTensorBoardLogDir(tensorboard_log_dir) def load(self, weights=None, sep='/', ignore_weights=None, eager=False): """ @@ -831,32 +826,6 @@ class Caffe2DML(BaseSystemMLClassifier): else: raise TypeError('Please use spark session of type pyspark.sql.session.SparkSession in the constructor') - def visualize(self, layerName=None, varType='weight', aggFn='mean'): - """ - Use this to visualize the training procedure (requires validation_percentage to be non-zero). - When one provides no argument to this method, we visualize training and validation loss. - - Parameters - ---------- - layerName: Name of the layer in the Caffe prototype - varType: should be either 'weight', 'bias', 'dweight', 'dbias', 'output' or 'doutput' - aggFn: should be either 'sum', 'mean', 'var' or 'sd' - """ - valid_vis_var_types = ['weight', 'bias', 'dweight', 'dbias', 'output', 'doutput'] - valid_vis_aggFn = [ 'sum', 'mean', 'var', 'sd' ] - if layerName is not None and varType is not None and aggFn is not None: - # Visualize other values - if varType not in valid_vis_var_types: - raise ValueError('The second argument should be either weight, bias, dweight, dbias, output or doutput') - if aggFn not in valid_vis_aggFn: - raise ValueError('The third argument should be either sum, mean, var, sd.') - if self.visualize_called: - self.estimator.visualizeLoss() - self.estimator.visualizeLayer(layerName, varType, aggFn) - else: - self.estimator.visualizeLoss() - self.visualize_called = True - return self class Keras2DML(Caffe2DML): """ http://git-wip-us.apache.org/repos/asf/systemml/blob/a4962f39/src/main/scala/org/apache/sysml/api/dl/Caffe2DML.scala ---------------------------------------------------------------------- diff --git a/src/main/scala/org/apache/sysml/api/dl/Caffe2DML.scala b/src/main/scala/org/apache/sysml/api/dl/Caffe2DML.scala index 91373bc..03b9a3b 100644 --- a/src/main/scala/org/apache/sysml/api/dl/Caffe2DML.scala +++ b/src/main/scala/org/apache/sysml/api/dl/Caffe2DML.scala @@ -303,9 +303,6 @@ class Caffe2DML(val sc: SparkContext, // TODO: throw error or warning if user tries to set solver_mode == GPU instead of using setGPU method - // Method called by Python mllearn to visualize variable of certain layer - def visualizeLayer(layerName: String, varType: String, aggFn: String): Unit = visualizeLayer(net, layerName, varType, aggFn) - def getTrainAlgo(): String = if (inputs.containsKey("$train_algo")) inputs.get("$train_algo") else "minibatch" def getTestAlgo(): String = if (inputs.containsKey("$test_algo")) inputs.get("$test_algo") else "minibatch" @@ -374,9 +371,6 @@ class Caffe2DML(val sc: SparkContext, } // ---------------------------------------------------------------------------- - // Check if this is necessary - if (doVisualize) tabDMLScript.append("print(" + asDMLString("Visualization counter:") + " + viz_counter)") - val trainingScript = tabDMLScript.toString() // Print script generation time and the DML script on stdout System.out.println("Time taken to generate training script from Caffe proto: " + ((System.nanoTime() - startTrainingTime) * 1e-9) + " seconds.") @@ -522,7 +516,6 @@ class Caffe2DML(val sc: SparkContext, tabDMLScript.append( print(dmlConcat(asDMLString("Iter:"), "iter", asDMLString(", training loss:"), "training_loss", asDMLString(", training accuracy:"), "training_accuracy")) ) - appendTrainingVisualizationBody(dmlScript, numTabs) printClassificationReport } } else { @@ -615,7 +608,6 @@ class Caffe2DML(val sc: SparkContext, tabDMLScript.append( print(dmlConcat(asDMLString("Iter:"), "iter", asDMLString(", validation loss:"), "validation_loss", asDMLString(", validation accuracy:"), "validation_accuracy")) ) - appendValidationVisualizationBody(dmlScript, numTabs) } } } http://git-wip-us.apache.org/repos/asf/systemml/blob/a4962f39/src/main/scala/org/apache/sysml/api/dl/DMLGenerator.scala ---------------------------------------------------------------------- diff --git a/src/main/scala/org/apache/sysml/api/dl/DMLGenerator.scala b/src/main/scala/org/apache/sysml/api/dl/DMLGenerator.scala index b68d493..304f788 100644 --- a/src/main/scala/org/apache/sysml/api/dl/DMLGenerator.scala +++ b/src/main/scala/org/apache/sysml/api/dl/DMLGenerator.scala @@ -188,81 +188,8 @@ trait NextBatchGenerator extends TabbedDMLGenerator { assignBatch(tabDMLScript, "Xb", Caffe2DML.XVal, "yb", Caffe2DML.yVal, "", Caffe2DML.numValidationImages, "iVal") } -trait VisualizeDMLGenerator extends TabbedDMLGenerator { - var doVisualize = false - var _tensorboardLogDir: String = null - def setTensorBoardLogDir(log: String): Unit = _tensorboardLogDir = log - def tensorboardLogDir: String = { - if (_tensorboardLogDir == null) { - _tensorboardLogDir = java.io.File.createTempFile("temp", System.nanoTime().toString()).getAbsolutePath - } - _tensorboardLogDir - } - def visualizeLoss(): Unit = { - checkTensorBoardDependency() - doVisualize = true - // Visualize for both training and validation - visualize(" ", " ", "training_loss", "iter", "training_loss", true) - visualize(" ", " ", "training_accuracy", "iter", "training_accuracy", true) - visualize(" ", " ", "validation_loss", "iter", "validation_loss", false) - visualize(" ", " ", "validation_accuracy", "iter", "validation_accuracy", false) - } - val visTrainingDMLScript: StringBuilder = new StringBuilder - val visValidationDMLScript: StringBuilder = new StringBuilder - def checkTensorBoardDependency(): Unit = - try { - if (!doVisualize) - Class.forName("com.google.protobuf.GeneratedMessageV3") - } catch { - case _: ClassNotFoundException => - throw new DMLRuntimeException( - "To use visualize() feature, you will have to include protobuf-java-3.2.0.jar in your classpath. Hint: you can download the jar from http://central.maven.org/maven2/com/google/protobuf/protobuf-java/3.2.0/protobuf-java-3.2.0.jar"; - ) - } - private def visualize(layerName: String, varType: String, aggFn: String, x: String, y: String, isTraining: Boolean) = { - val dmlScript = if (isTraining) visTrainingDMLScript else visValidationDMLScript - dmlScript.append( - "viz_counter1 = visualize(" + - commaSep(asDMLString(layerName), asDMLString(varType), asDMLString(aggFn), x, y, asDMLString(tensorboardLogDir)) - + ");\n" - ) - dmlScript.append("viz_counter = viz_counter + viz_counter1\n") - } - def visualizeLayer(net: CaffeNetwork, layerName: String, varType: String, aggFn: String): Unit = { - // 'weight', 'bias', 'dweight', 'dbias', 'output' or 'doutput' - // 'sum', 'mean', 'var' or 'sd' - checkTensorBoardDependency() - doVisualize = true - if (net.getLayers.filter(_.equals(layerName)).size == 0) - throw new DMLRuntimeException("Cannot visualize the layer:" + layerName) - val dmlVar = { - val l = net.getCaffeLayer(layerName) - varType match { - case "weight" => l.weight - case "bias" => l.bias - case "dweight" => l.dWeight - case "dbias" => l.dBias - case "output" => l.out - // case "doutput" => l.dX - case _ => throw new DMLRuntimeException("Cannot visualize the variable of type:" + varType) - } - } - if (dmlVar == null) - throw new DMLRuntimeException("Cannot visualize the variable of type:" + varType) - // Visualize for both training and validation - visualize(layerName, varType, aggFn, "iter", aggFn + "(" + dmlVar + ")", true) - visualize(layerName, varType, aggFn, "iter", aggFn + "(" + dmlVar + ")", false) - } - - def appendTrainingVisualizationBody(dmlScript: StringBuilder, numTabs: Int): Unit = - if (doVisualize) - tabDMLScript(dmlScript, numTabs).append(visTrainingDMLScript.toString) - def appendValidationVisualizationBody(dmlScript: StringBuilder, numTabs: Int): Unit = - if (doVisualize) - tabDMLScript(dmlScript, numTabs).append(visValidationDMLScript.toString) -} -trait DMLGenerator extends SourceDMLGenerator with NextBatchGenerator with VisualizeDMLGenerator { +trait DMLGenerator extends SourceDMLGenerator with NextBatchGenerator { // Also makes "code reading" possible for Caffe2DML :) var dmlScript = new StringBuilder var numTabs = 0 @@ -270,9 +197,6 @@ trait DMLGenerator extends SourceDMLGenerator with NextBatchGenerator with Visua dmlScript.clear() alreadyImported.clear() numTabs = 0 - visTrainingDMLScript.clear() - visValidationDMLScript.clear() - doVisualize = false } // ------------------------------------------------------------------------------------------------- // Helper functions that calls super class methods and simplifies the code of this trait @@ -341,16 +265,7 @@ trait DMLGenerator extends SourceDMLGenerator with NextBatchGenerator with Visua if (isTraining) { // Append external built-in function headers: - // 1. visualize external built-in function header - if (doVisualize) { - tabDMLScript.append( - "visualize = externalFunction(String layerName, String varType, String aggFn, Double x, Double y, String logDir) return (Double B) " + - "implemented in (classname=\"org.apache.sysml.udf.lib.Caffe2DMLVisualizeWrapper\",exectype=\"mem\"); \n" - ) - tabDMLScript.append("viz_counter = 0\n") - System.out.println("Please use the following command for visualizing: tensorboard --logdir=" + tensorboardLogDir) - } - // 2. update_nesterov external built-in function header + // 1. update_nesterov external built-in function header if (Caffe2DML.USE_NESTEROV_UDF) { tabDMLScript.append( "update_nesterov = externalFunction(matrix[double] X, matrix[double] dX, double lr, double mu, matrix[double] v, double lambda) return (matrix[double] X, matrix[double] v) implemented in (classname=\"org.apache.sysml.udf.lib.SGDNesterovUpdate\",exectype=\"mem\"); \n"
