This is an automated email from the ASF dual-hosted git repository.
zhasheng pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-mxnet.git
The following commit(s) were added to refs/heads/master by this push:
new 3ba0c22 fix example (#8006)
3ba0c22 is described below
commit 3ba0c22a897ae2598b4025c52fe021096c31a181
Author: Sheng Zha <[email protected]>
AuthorDate: Tue Sep 26 15:18:34 2017 -0700
fix example (#8006)
* fix example
* remove unnecessary path
---
example/adversary/adversary_generation.ipynb | 104 ++++++---------------------
example/adversary/data.py | 49 -------------
example/multi-task/data.py | 49 -------------
example/multi-task/example_multi_task.py | 45 +++++++++++-
example/numpy-ops/custom_softmax.py | 5 +-
example/numpy-ops/data.py | 49 -------------
example/numpy-ops/ndarray_softmax.py | 6 +-
example/numpy-ops/numpy_softmax.py | 6 +-
tests/python/common/get_data.py | 57 +++++++++++----
9 files changed, 120 insertions(+), 250 deletions(-)
diff --git a/example/adversary/adversary_generation.ipynb
b/example/adversary/adversary_generation.ipynb
index 3af733f..a545187 100644
--- a/example/adversary/adversary_generation.ipynb
+++ b/example/adversary/adversary_generation.ipynb
@@ -15,7 +15,7 @@
},
{
"cell_type": "code",
- "execution_count": 1,
+ "execution_count": null,
"metadata": {
"collapsed": false
},
@@ -28,7 +28,10 @@
"import matplotlib.pyplot as plt\n",
"import matplotlib.cm as cm\n",
"\n",
- "from data import mnist_iterator"
+ "import os\n",
+ "import sys\n",
+ "sys.path.append(os.path.join(os.getcwd(),
\"../../tests/python/common\"))\n",
+ "from get_data import mnist_iterator"
]
},
{
@@ -42,20 +45,20 @@
},
{
"cell_type": "code",
- "execution_count": 2,
+ "execution_count": null,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
- "dev = mx.gpu()\n",
+ "dev = mx.cpu()\n",
"batch_size = 100\n",
"train_iter, val_iter = mnist_iterator(batch_size=batch_size, input_shape
= (1,28,28))"
]
},
{
"cell_type": "code",
- "execution_count": 3,
+ "execution_count": null,
"metadata": {
"collapsed": true
},
@@ -83,7 +86,7 @@
},
{
"cell_type": "code",
- "execution_count": 4,
+ "execution_count": null,
"metadata": {
"collapsed": true
},
@@ -99,7 +102,7 @@
},
{
"cell_type": "code",
- "execution_count": 5,
+ "execution_count": null,
"metadata": {
"collapsed": true
},
@@ -108,7 +111,7 @@
"def LogLossGrad(alpha, label):\n",
" grad = np.copy(alpha)\n",
" for i in range(alpha.shape[0]):\n",
- " grad[i, label[i]] -= 1.\n",
+ " grad[i, int(label[i])] -= 1.\n",
" return grad"
]
},
@@ -121,7 +124,7 @@
},
{
"cell_type": "code",
- "execution_count": 6,
+ "execution_count": null,
"metadata": {
"collapsed": false
},
@@ -151,7 +154,7 @@
},
{
"cell_type": "code",
- "execution_count": 7,
+ "execution_count": null,
"metadata": {
"collapsed": true
},
@@ -165,7 +168,7 @@
},
{
"cell_type": "code",
- "execution_count": 8,
+ "execution_count": null,
"metadata": {
"collapsed": true
},
@@ -181,7 +184,7 @@
"def CalLoss(pred_prob, label):\n",
" loss = 0.\n",
" for i in range(pred_prob.shape[0]):\n",
- " loss += -np.log(max(pred_prob[i, label[i]], 1e-10))\n",
+ " loss += -np.log(max(pred_prob[i, int(label[i])], 1e-10))\n",
" return loss"
]
},
@@ -194,30 +197,11 @@
},
{
"cell_type": "code",
- "execution_count": 9,
+ "execution_count": null,
"metadata": {
"collapsed": false
},
- "outputs": [
- {
- "name": "stderr",
- "output_type": "stream",
- "text": [
- "/usr/local/lib/python2.7/dist-packages/ipykernel/__main__.py:11:
VisibleDeprecationWarning: using a non-integer number instead of an integer
will result in an error in the future\n",
- "/usr/local/lib/python2.7/dist-packages/ipykernel/__main__.py:4:
VisibleDeprecationWarning: using a non-integer number instead of an integer
will result in an error in the future\n"
- ]
- },
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Train Accuracy: 0.92\t Train Loss: 0.28074\n",
- "Train Accuracy: 0.98\t Train Loss: 0.08431\n",
- "Train Accuracy: 0.98\t Train Loss: 0.05848\n",
- "Train Accuracy: 0.99\t Train Loss: 0.04575\n"
- ]
- }
- ],
+ "outputs": [],
"source": [
"num_round = 4\n",
"train_acc = 0.\n",
@@ -260,27 +244,11 @@
},
{
"cell_type": "code",
- "execution_count": 10,
+ "execution_count": null,
"metadata": {
"collapsed": false
},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "('Val Batch Accuracy: ', 0.98999999999999999)\n",
- "('Val Batch Accuracy after pertubation: ', 0.02)\n"
- ]
- },
- {
- "name": "stderr",
- "output_type": "stream",
- "text": [
- "/usr/local/lib/python2.7/dist-packages/ipykernel/__main__.py:4:
VisibleDeprecationWarning: using a non-integer number instead of an integer
will result in an error in the future\n"
- ]
- }
- ],
+ "outputs": [],
"source": [
"val_iter.reset()\n",
"batch = val_iter.next()\n",
@@ -313,30 +281,11 @@
},
{
"cell_type": "code",
- "execution_count": 11,
+ "execution_count": null,
"metadata": {
"collapsed": false
},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "true: 5\n",
- "pred: 3\n"
- ]
- },
- {
- "data": {
- "image/png":
"iVBORw0KGgoAAAANSUhEUgAAAWEAAAFfCAYAAACfj30KAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAAPYQAAD2EBqD+naQAAIABJREFUeJztvW2sLctZHvjU3mt/nHttXyAefPENEUYYwUWIkcxoRiaW\niWYMGUCE/EkEKBhGg8KQHyijIIQJg8XE+ZEoAcmA0CAGgZQggfiMxJdFiAfGMZMwIopzpSEOEBNf\n7gXjxL73nLP3Wnuvnh/nvOs861nvW121vrrX2vVIra6u7tWrurrqqbfej+rUdR0aGhoaGobBydAF\naGhoaLjLaCTc0NDQMCAaCTc0NDQMiEbCDQ0NDQOikXBDQ0PDgGgk3NDQ0DAgGgk3NDQ0DIhGwg0N\nDQ0DopFwQ0NDw4CYDF2AlNKfA/DlAP4QwNWwpWloaGjYCi4BfBaAX+267s9yF+6MhFNK3wrg7wD4\nDAAf
[...]
- "text/plain": [
- "<matplotlib.figure.Figure at 0x7f61ec17dd50>"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- }
- ],
+ "outputs": [],
"source": [
"import random as rnd\n",
"idx = rnd.randint(0, 99)\n",
@@ -345,15 +294,6 @@
"print(\"true: %d\" % label.asnumpy()[idx])\n",
"print(\"pred: %d\" % np.argmax(pred, axis=1)[idx])"
]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "collapsed": true
- },
- "outputs": [],
- "source": []
}
],
"metadata": {
@@ -372,7 +312,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython2",
- "version": "2.7.6"
+ "version": "2.7.13"
}
},
"nbformat": 4,
diff --git a/example/adversary/data.py b/example/adversary/data.py
deleted file mode 100644
index 0ca8e1f..0000000
--- a/example/adversary/data.py
+++ /dev/null
@@ -1,49 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-# pylint: skip-file
-""" data iterator for mnist """
-import sys
-import os
-# code to automatically download dataset
-curr_path = os.path.dirname(os.path.abspath(os.path.expanduser(__file__)))
-sys.path.append(os.path.join(curr_path, "../../tests/python/common"))
-import get_data
-import mxnet as mx
-
-def mnist_iterator(batch_size, input_shape):
- """return train and val iterators for mnist"""
- # download data
- get_data.GetMNIST_ubyte()
- flat = False if len(input_shape) == 3 else True
-
- train_dataiter = mx.io.MNISTIter(
- image="data/train-images-idx3-ubyte",
- label="data/train-labels-idx1-ubyte",
- input_shape=input_shape,
- batch_size=batch_size,
- shuffle=True,
- flat=flat)
-
- val_dataiter = mx.io.MNISTIter(
- image="data/t10k-images-idx3-ubyte",
- label="data/t10k-labels-idx1-ubyte",
- input_shape=input_shape,
- batch_size=batch_size,
- flat=flat)
-
- return (train_dataiter, val_dataiter)
diff --git a/example/multi-task/data.py b/example/multi-task/data.py
deleted file mode 100644
index 0ca8e1f..0000000
--- a/example/multi-task/data.py
+++ /dev/null
@@ -1,49 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-# pylint: skip-file
-""" data iterator for mnist """
-import sys
-import os
-# code to automatically download dataset
-curr_path = os.path.dirname(os.path.abspath(os.path.expanduser(__file__)))
-sys.path.append(os.path.join(curr_path, "../../tests/python/common"))
-import get_data
-import mxnet as mx
-
-def mnist_iterator(batch_size, input_shape):
- """return train and val iterators for mnist"""
- # download data
- get_data.GetMNIST_ubyte()
- flat = False if len(input_shape) == 3 else True
-
- train_dataiter = mx.io.MNISTIter(
- image="data/train-images-idx3-ubyte",
- label="data/train-labels-idx1-ubyte",
- input_shape=input_shape,
- batch_size=batch_size,
- shuffle=True,
- flat=flat)
-
- val_dataiter = mx.io.MNISTIter(
- image="data/t10k-images-idx3-ubyte",
- label="data/t10k-labels-idx1-ubyte",
- input_shape=input_shape,
- batch_size=batch_size,
- flat=flat)
-
- return (train_dataiter, val_dataiter)
diff --git a/example/multi-task/example_multi_task.py
b/example/multi-task/example_multi_task.py
index 853d435..5a42516 100644
--- a/example/multi-task/example_multi_task.py
+++ b/example/multi-task/example_multi_task.py
@@ -17,8 +17,11 @@
# pylint: skip-file
import sys
+import os
sys.path.insert(0, "../../python/")
-from data import mnist_iterator
+curr_path = os.path.dirname(os.path.abspath(os.path.expanduser(__file__)))
+sys.path.append(os.path.join(curr_path, "../../tests/python/common"))
+from get_data import mnist_iterator
import mxnet as mx
import numpy as np
import logging
@@ -76,7 +79,13 @@ class Multi_Accuracy(mx.metric.EvalMetric):
"""Calculate accuracies of multi label"""
def __init__(self, num=None):
- super(Multi_Accuracy, self).__init__('multi-accuracy', num)
+ self.num = num
+ super(Multi_Accuracy, self).__init__('multi-accuracy')
+
+ def reset(self):
+ """Resets the internal evaluation result to initial state."""
+ self.num_inst = 0 if self.num is None else [0] * self.num
+ self.sum_metric = 0.0 if self.num is None else [0.0] * self.num
def update(self, labels, preds):
mx.metric.check_label_shapes(labels, preds)
@@ -90,13 +99,43 @@ class Multi_Accuracy(mx.metric.EvalMetric):
mx.metric.check_label_shapes(label, pred_label)
- if i is None:
+ if self.num is None:
self.sum_metric += (pred_label.flat == label.flat).sum()
self.num_inst += len(pred_label.flat)
else:
self.sum_metric[i] += (pred_label.flat == label.flat).sum()
self.num_inst[i] += len(pred_label.flat)
+ def get(self):
+ """Gets the current evaluation result.
+
+ Returns
+ -------
+ names : list of str
+ Name of the metrics.
+ values : list of float
+ Value of the evaluations.
+ """
+ if self.num is None:
+ return super(Multi_Accuracy, self).get()
+ else:
+ return zip(*(('%s-task%d'%(self.name, i), float('nan') if
self.num_inst[i] == 0
+ else self.sum_metric[i]
/ self.num_inst[i])
+ for i in range(self.num)))
+
+ def get_name_value(self):
+ """Returns zipped name and value pairs.
+
+ Returns
+ -------
+ list of tuples
+ A (name, value) tuple list.
+ """
+ if self.num is None:
+ return super(Multi_Accuracy, self).get_name_value()
+ name, value = self.get()
+ return list(zip(name, value))
+
batch_size=100
num_epochs=100
diff --git a/example/numpy-ops/custom_softmax.py
b/example/numpy-ops/custom_softmax.py
index 162215f..cbfcdfa 100644
--- a/example/numpy-ops/custom_softmax.py
+++ b/example/numpy-ops/custom_softmax.py
@@ -16,8 +16,11 @@
# under the License.
# pylint: skip-file
+import sys
import os
-from data import mnist_iterator
+curr_path = os.path.dirname(os.path.abspath(os.path.expanduser(__file__)))
+sys.path.append(os.path.join(curr_path, "../../tests/python/common"))
+from get_data import mnist_iterator
import mxnet as mx
import numpy as np
import logging
diff --git a/example/numpy-ops/data.py b/example/numpy-ops/data.py
deleted file mode 100644
index 0ca8e1f..0000000
--- a/example/numpy-ops/data.py
+++ /dev/null
@@ -1,49 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-# pylint: skip-file
-""" data iterator for mnist """
-import sys
-import os
-# code to automatically download dataset
-curr_path = os.path.dirname(os.path.abspath(os.path.expanduser(__file__)))
-sys.path.append(os.path.join(curr_path, "../../tests/python/common"))
-import get_data
-import mxnet as mx
-
-def mnist_iterator(batch_size, input_shape):
- """return train and val iterators for mnist"""
- # download data
- get_data.GetMNIST_ubyte()
- flat = False if len(input_shape) == 3 else True
-
- train_dataiter = mx.io.MNISTIter(
- image="data/train-images-idx3-ubyte",
- label="data/train-labels-idx1-ubyte",
- input_shape=input_shape,
- batch_size=batch_size,
- shuffle=True,
- flat=flat)
-
- val_dataiter = mx.io.MNISTIter(
- image="data/t10k-images-idx3-ubyte",
- label="data/t10k-labels-idx1-ubyte",
- input_shape=input_shape,
- batch_size=batch_size,
- flat=flat)
-
- return (train_dataiter, val_dataiter)
diff --git a/example/numpy-ops/ndarray_softmax.py
b/example/numpy-ops/ndarray_softmax.py
index aa8555e..ed60e8e 100644
--- a/example/numpy-ops/ndarray_softmax.py
+++ b/example/numpy-ops/ndarray_softmax.py
@@ -16,7 +16,11 @@
# under the License.
# pylint: skip-file
-from data import mnist_iterator
+import os
+import sys
+curr_path = os.path.dirname(os.path.abspath(os.path.expanduser(__file__)))
+sys.path.append(os.path.join(curr_path, "../../tests/python/common"))
+from get_data import mnist_iterator
import mxnet as mx
import numpy as np
import logging
diff --git a/example/numpy-ops/numpy_softmax.py
b/example/numpy-ops/numpy_softmax.py
index f90783b..1650fe4 100644
--- a/example/numpy-ops/numpy_softmax.py
+++ b/example/numpy-ops/numpy_softmax.py
@@ -16,7 +16,11 @@
# under the License.
# pylint: skip-file
-from data import mnist_iterator
+import sys
+import os
+curr_path = os.path.dirname(os.path.abspath(os.path.expanduser(__file__)))
+sys.path.append(os.path.join(curr_path, "../../tests/python/common"))
+from get_data import mnist_iterator
import mxnet as mx
import numpy as np
import logging
diff --git a/tests/python/common/get_data.py b/tests/python/common/get_data.py
index 35482f8..3cd39a1 100644
--- a/tests/python/common/get_data.py
+++ b/tests/python/common/get_data.py
@@ -19,36 +19,63 @@
import os, gzip
import pickle as pickle
import sys
+from mxnet.test_utils import download
+import zipfile
+import mxnet as mx
# download mnist.pkl.gz
def GetMNIST_pkl():
- if not os.path.isdir("data/"):
- os.system("mkdir data/")
+ if not os.path.isdir("data"):
+ os.makedirs('data')
if not os.path.exists('data/mnist.pkl.gz'):
- os.system("wget -q http://deeplearning.net/data/mnist/mnist.pkl.gz -P
data/")
+ download('http://deeplearning.net/data/mnist/mnist.pkl.gz',
+ dirname='data')
# download ubyte version of mnist and untar
def GetMNIST_ubyte():
- if not os.path.isdir("data/"):
- os.system("mkdir data/")
+ if not os.path.isdir("data"):
+ os.makedirs('data')
if (not os.path.exists('data/train-images-idx3-ubyte')) or \
(not os.path.exists('data/train-labels-idx1-ubyte')) or \
(not os.path.exists('data/t10k-images-idx3-ubyte')) or \
(not os.path.exists('data/t10k-labels-idx1-ubyte')):
- os.system("wget -q http://data.mxnet.io/mxnet/data/mnist.zip -P data/")
- os.chdir("./data")
- os.system("unzip -u mnist.zip")
- os.chdir("..")
+ zip_file_path = download('http://data.mxnet.io/mxnet/data/mnist.zip',
+ dirname='data')
+ with zipfile.ZipFile(zip_file_path) as zf:
+ zf.extractall('data')
# download cifar
def GetCifar10():
- if not os.path.isdir("data/"):
- os.system("mkdir data/")
+ if not os.path.isdir("data"):
+ os.makedirs('data')
if (not os.path.exists('data/cifar/train.rec')) or \
(not os.path.exists('data/cifar/test.rec')) or \
(not os.path.exists('data/cifar/train.lst')) or \
(not os.path.exists('data/cifar/test.lst')):
- os.system("wget -q http://data.mxnet.io/mxnet/data/cifar10.zip -P
data/")
- os.chdir("./data")
- os.system("unzip -u cifar10.zip")
- os.chdir("..")
+ zip_file_path = download('http://data.mxnet.io/mxnet/data/cifar10.zip',
+ dirname='data')
+ with zipfile.ZipFile(zip_file_path) as zf:
+ zf.extractall('data')
+
+def mnist_iterator(batch_size, input_shape):
+ """return train and val iterators for mnist"""
+ # download data
+ GetMNIST_ubyte()
+ flat = False if len(input_shape) == 3 else True
+
+ train_dataiter = mx.io.MNISTIter(
+ image="data/train-images-idx3-ubyte",
+ label="data/train-labels-idx1-ubyte",
+ input_shape=input_shape,
+ batch_size=batch_size,
+ shuffle=True,
+ flat=flat)
+
+ val_dataiter = mx.io.MNISTIter(
+ image="data/t10k-images-idx3-ubyte",
+ label="data/t10k-labels-idx1-ubyte",
+ input_shape=input_shape,
+ batch_size=batch_size,
+ flat=flat)
+
+ return (train_dataiter, val_dataiter)
--
To stop receiving notification emails like this one, please contact
['"[email protected]" <[email protected]>'].
