Finish precision and recall for multi-label metrics Implement recall and precision computation via numpy
Project: http://git-wip-us.apache.org/repos/asf/incubator-singa/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-singa/commit/de8a6541 Tree: http://git-wip-us.apache.org/repos/asf/incubator-singa/tree/de8a6541 Diff: http://git-wip-us.apache.org/repos/asf/incubator-singa/diff/de8a6541 Branch: refs/heads/master Commit: de8a6541c29a4d9cd39114f85dbcc1cb0187af59 Parents: dde8d14 Author: RUAN0007 <ruanpingch...@gmail.com> Authored: Wed Feb 22 21:43:56 2017 +0800 Committer: RUAN0007 <ruanpingch...@gmail.com> Committed: Wed Feb 22 21:50:44 2017 +0800 ---------------------------------------------------------------------- python/singa/metric.py | 78 +++++++++++++++++++++++++++++++++++++---- test/python/test_metric.py | 28 +++++++++++++++ 2 files changed, 100 insertions(+), 6 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/de8a6541/python/singa/metric.py ---------------------------------------------------------------------- diff --git a/python/singa/metric.py b/python/singa/metric.py index 2492965..6ddcd27 100644 --- a/python/singa/metric.py +++ b/python/singa/metric.py @@ -180,20 +180,86 @@ class Precision(Metric): pred_np = np.argsort(-x_np)[:,0:self.top_k] #Sort in descending order - tmp_np = np.zeros(pred_np.shape, dtype=np.float32) + prcs_np = np.zeros(pred_np.shape[0], dtype=np.float32) for i in range(pred_np.shape[0]): - tmp_np[i] = y_np[i,pred_np[i]] + #groundtruth labels + label_np = np.argwhere(y_np[i]) - prcs_np = np.average(tmp_np, axis=1) + #Num of common labels among prediction and groundtruth + num_intersect = np.intersect1d(pred_np[i], label_np).size + prcs_np[i] = num_intersect / float(self.top_k) - prcs = tensor.from_numpy(prcs_np) + precision = tensor.from_numpy(prcs_np) x.to_device(dev) y.to_device(dev) - prcs.to_device(dev) + precision.to_device(dev) - return prcs + return precision + + + def evaluate(self, x, y): + '''Compute the averaged precision over all samples. + + Args: + x (Tensor): predictions, one row per sample + y (Tensor): ground truth values, one row per sample + Returns: + a float value for the averaged metric + ''' + + return tensor.average(self.forward(x,y)) + + +class Recall(Metric): + '''Make the top-k labels of max probability as the prediction + + Compute the recall against the groundtruth labels + ''' + def __init__(self, top_k): + self.top_k = top_k + + + def forward(self, x, y): + '''Compute the recall for each sample. + + Convert tensor to numpy for computation + + Args: + x (Tensor): predictions, one row per sample + y (Tensor): ground truth labels, one row per sample + + Returns: + a tensor of floats, one per sample + ''' + + dev = x.device + x.to_host() + y.to_host() + + x_np = tensor.to_numpy(x) + y_np = tensor.to_numpy(y) + + pred_np = np.argsort(-x_np)[:,0:self.top_k] #Sort in descending order + + recall_np = np.zeros(pred_np.shape[0], dtype=np.float32) + + for i in range(pred_np.shape[0]): + #Return the index of non-zero dimension of i-th sample + label_np = np.argwhere(y_np[i]) + + #Num of common labels among prediction and groundtruth + num_intersect = np.intersect1d(pred_np[i], label_np).size + recall_np[i] = float(num_intersect) / label_np.size + + recall = tensor.from_numpy(recall_np) + + x.to_device(dev) + y.to_device(dev) + recall.to_device(dev) + + return recall def evaluate(self, x, y): http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/de8a6541/test/python/test_metric.py ---------------------------------------------------------------------- diff --git a/test/python/test_metric.py b/test/python/test_metric.py index 0d298ae..e7a51c3 100644 --- a/test/python/test_metric.py +++ b/test/python/test_metric.py @@ -52,5 +52,33 @@ class TestPrecision(unittest.TestCase): e = self.prcs.evaluate(self.x,self.y) self.assertAlmostEqual(e, (0.5 + 1 + 0) / 3) +class TestRecall(unittest.TestCase): + def setUp(self): + x_np = np.asarray([[0.7, 0.2, 0.1], + [0.2, 0.4, 0.5], + [0.2,0.4,0.4]], + dtype=np.float32) + + y_np = np.asarray([[1, 0, 1], + [1, 1, 1], + [1, 0, 0]], + dtype=np.int32) + + self.recall = metric.Recall(top_k=2) + self.x = tensor.from_numpy(x_np) + self.y = tensor.from_numpy(y_np) + + + def test_forward(self): + r = self.recall.forward(self.x,self.y) + self.assertAlmostEqual(tensor.to_numpy(r)[0], 0.5) + self.assertAlmostEqual(tensor.to_numpy(r)[1], 2.0 / 3) + self.assertAlmostEqual(tensor.to_numpy(r)[2], 0) + + + def test_evaluate(self): + e = self.recall.evaluate(self.x,self.y) + self.assertAlmostEqual(e, (0.5 + 2.0 / 3 + 0) / 3) + if __name__ == '__main__': unittest.main()
