Updated Branches: refs/heads/trunk b3eada3c0 -> fad298ebb
GIRAPH-786: XSparseVector create a lot of objects in add/write (ikabiljo via majakabiljo) Project: http://git-wip-us.apache.org/repos/asf/giraph/repo Commit: http://git-wip-us.apache.org/repos/asf/giraph/commit/fad298eb Tree: http://git-wip-us.apache.org/repos/asf/giraph/tree/fad298eb Diff: http://git-wip-us.apache.org/repos/asf/giraph/diff/fad298eb Branch: refs/heads/trunk Commit: fad298ebb5a35c7466aac69af57f60775ff6a87f Parents: b3eada3 Author: Maja Kabiljo <[email protected]> Authored: Thu Oct 17 18:31:31 2013 -0700 Committer: Maja Kabiljo <[email protected]> Committed: Thu Oct 17 18:31:31 2013 -0700 ---------------------------------------------------------------------- CHANGELOG | 3 +++ .../matrix/sparse/DoubleSparseVector.java | 12 ++++++++++-- .../matrix/sparse/FloatSparseVector.java | 12 ++++++++++-- .../matrix/sparse/IntSparseVector.java | 12 ++++++++++-- .../matrix/sparse/LongSparseVector.java | 18 +++++++++++++----- 5 files changed, 46 insertions(+), 11 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/giraph/blob/fad298eb/CHANGELOG ---------------------------------------------------------------------- diff --git a/CHANGELOG b/CHANGELOG index f457026..9d55b47 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,6 +1,9 @@ Giraph Change Log Release 1.1.0 - unreleased + GIRAPH-786: XSparseVector create a lot of objects in add/write + (ikabiljo via majakabiljo) + GIRAPH-773: Cannot use only EdgeOutputFormat (armax00 via claudio) GIRAPH-779: Make it easier to reuse objects with hive-io input (majakabiljo) http://git-wip-us.apache.org/repos/asf/giraph/blob/fad298eb/giraph-core/src/main/java/org/apache/giraph/aggregators/matrix/sparse/DoubleSparseVector.java ---------------------------------------------------------------------- diff --git a/giraph-core/src/main/java/org/apache/giraph/aggregators/matrix/sparse/DoubleSparseVector.java b/giraph-core/src/main/java/org/apache/giraph/aggregators/matrix/sparse/DoubleSparseVector.java index de119fb..fb54459 100644 --- a/giraph-core/src/main/java/org/apache/giraph/aggregators/matrix/sparse/DoubleSparseVector.java +++ b/giraph-core/src/main/java/org/apache/giraph/aggregators/matrix/sparse/DoubleSparseVector.java @@ -19,11 +19,13 @@ package org.apache.giraph.aggregators.matrix.sparse; import it.unimi.dsi.fastutil.ints.Int2DoubleMap; + import java.io.DataInput; import java.io.DataOutput; import java.io.IOException; import it.unimi.dsi.fastutil.ints.Int2DoubleOpenHashMap; +import it.unimi.dsi.fastutil.objects.ObjectIterator; import org.apache.hadoop.io.Writable; @@ -96,7 +98,10 @@ public class DoubleSparseVector implements Writable { * @param other the vector to add. */ public void add(DoubleSparseVector other) { - for (Int2DoubleMap.Entry entry : other.entries.int2DoubleEntrySet()) { + ObjectIterator<Int2DoubleMap.Entry> iter = + other.entries.int2DoubleEntrySet().fastIterator(); + while (iter.hasNext()) { + Int2DoubleMap.Entry entry = iter.next(); entries.addTo(entry.getIntKey(), entry.getDoubleValue()); } } @@ -104,7 +109,10 @@ public class DoubleSparseVector implements Writable { @Override public void write(DataOutput out) throws IOException { out.writeInt(entries.size()); - for (Int2DoubleMap.Entry entry : entries.int2DoubleEntrySet()) { + ObjectIterator<Int2DoubleMap.Entry> iter = + entries.int2DoubleEntrySet().fastIterator(); + while (iter.hasNext()) { + Int2DoubleMap.Entry entry = iter.next(); out.writeInt(entry.getIntKey()); out.writeDouble(entry.getDoubleValue()); } http://git-wip-us.apache.org/repos/asf/giraph/blob/fad298eb/giraph-core/src/main/java/org/apache/giraph/aggregators/matrix/sparse/FloatSparseVector.java ---------------------------------------------------------------------- diff --git a/giraph-core/src/main/java/org/apache/giraph/aggregators/matrix/sparse/FloatSparseVector.java b/giraph-core/src/main/java/org/apache/giraph/aggregators/matrix/sparse/FloatSparseVector.java index 4e94217..77a9bd9 100644 --- a/giraph-core/src/main/java/org/apache/giraph/aggregators/matrix/sparse/FloatSparseVector.java +++ b/giraph-core/src/main/java/org/apache/giraph/aggregators/matrix/sparse/FloatSparseVector.java @@ -19,11 +19,13 @@ package org.apache.giraph.aggregators.matrix.sparse; import it.unimi.dsi.fastutil.ints.Int2FloatMap; + import java.io.DataInput; import java.io.DataOutput; import java.io.IOException; import it.unimi.dsi.fastutil.ints.Int2FloatOpenHashMap; +import it.unimi.dsi.fastutil.objects.ObjectIterator; import org.apache.hadoop.io.Writable; @@ -96,7 +98,10 @@ public class FloatSparseVector implements Writable { * @param other the vector to add. */ public void add(FloatSparseVector other) { - for (Int2FloatMap.Entry entry : other.entries.int2FloatEntrySet()) { + ObjectIterator<Int2FloatMap.Entry> iter = + other.entries.int2FloatEntrySet().fastIterator(); + while (iter.hasNext()) { + Int2FloatMap.Entry entry = iter.next(); entries.addTo(entry.getIntKey(), entry.getFloatValue()); } } @@ -104,7 +109,10 @@ public class FloatSparseVector implements Writable { @Override public void write(DataOutput out) throws IOException { out.writeInt(entries.size()); - for (Int2FloatMap.Entry entry : entries.int2FloatEntrySet()) { + ObjectIterator<Int2FloatMap.Entry> iter = + entries.int2FloatEntrySet().fastIterator(); + while (iter.hasNext()) { + Int2FloatMap.Entry entry = iter.next(); out.writeInt(entry.getIntKey()); out.writeFloat(entry.getFloatValue()); } http://git-wip-us.apache.org/repos/asf/giraph/blob/fad298eb/giraph-core/src/main/java/org/apache/giraph/aggregators/matrix/sparse/IntSparseVector.java ---------------------------------------------------------------------- diff --git a/giraph-core/src/main/java/org/apache/giraph/aggregators/matrix/sparse/IntSparseVector.java b/giraph-core/src/main/java/org/apache/giraph/aggregators/matrix/sparse/IntSparseVector.java index 2482bd2..0ed6bc3 100644 --- a/giraph-core/src/main/java/org/apache/giraph/aggregators/matrix/sparse/IntSparseVector.java +++ b/giraph-core/src/main/java/org/apache/giraph/aggregators/matrix/sparse/IntSparseVector.java @@ -19,11 +19,13 @@ package org.apache.giraph.aggregators.matrix.sparse; import it.unimi.dsi.fastutil.ints.Int2IntMap; + import java.io.DataInput; import java.io.DataOutput; import java.io.IOException; import it.unimi.dsi.fastutil.ints.Int2IntOpenHashMap; +import it.unimi.dsi.fastutil.objects.ObjectIterator; import org.apache.hadoop.io.Writable; @@ -96,7 +98,10 @@ public class IntSparseVector implements Writable { * @param other the vector to add. */ public void add(IntSparseVector other) { - for (Int2IntMap.Entry entry : other.entries.int2IntEntrySet()) { + ObjectIterator<Int2IntMap.Entry> iter = + other.entries.int2IntEntrySet().fastIterator(); + while (iter.hasNext()) { + Int2IntMap.Entry entry = iter.next(); entries.addTo(entry.getIntKey(), entry.getIntValue()); } } @@ -104,7 +109,10 @@ public class IntSparseVector implements Writable { @Override public void write(DataOutput out) throws IOException { out.writeInt(entries.size()); - for (Int2IntMap.Entry entry : entries.int2IntEntrySet()) { + ObjectIterator<Int2IntMap.Entry> iter = + entries.int2IntEntrySet().fastIterator(); + while (iter.hasNext()) { + Int2IntMap.Entry entry = iter.next(); out.writeInt(entry.getIntKey()); out.writeInt(entry.getIntValue()); } http://git-wip-us.apache.org/repos/asf/giraph/blob/fad298eb/giraph-core/src/main/java/org/apache/giraph/aggregators/matrix/sparse/LongSparseVector.java ---------------------------------------------------------------------- diff --git a/giraph-core/src/main/java/org/apache/giraph/aggregators/matrix/sparse/LongSparseVector.java b/giraph-core/src/main/java/org/apache/giraph/aggregators/matrix/sparse/LongSparseVector.java index 8ab52a9..6337215 100644 --- a/giraph-core/src/main/java/org/apache/giraph/aggregators/matrix/sparse/LongSparseVector.java +++ b/giraph-core/src/main/java/org/apache/giraph/aggregators/matrix/sparse/LongSparseVector.java @@ -19,11 +19,13 @@ package org.apache.giraph.aggregators.matrix.sparse; import it.unimi.dsi.fastutil.ints.Int2LongMap; + import java.io.DataInput; import java.io.DataOutput; import java.io.IOException; import it.unimi.dsi.fastutil.ints.Int2LongOpenHashMap; +import it.unimi.dsi.fastutil.objects.ObjectIterator; import org.apache.hadoop.io.Writable; @@ -96,17 +98,23 @@ public class LongSparseVector implements Writable { * @param other the vector to add. */ public void add(LongSparseVector other) { - for (Int2LongMap.Entry kv : other.entries.int2LongEntrySet()) { - entries.addTo(kv.getIntKey(), kv.getLongValue()); + ObjectIterator<Int2LongMap.Entry> iter = + other.entries.int2LongEntrySet().fastIterator(); + while (iter.hasNext()) { + Int2LongMap.Entry entry = iter.next(); + entries.addTo(entry.getIntKey(), entry.getLongValue()); } } @Override public void write(DataOutput out) throws IOException { out.writeInt(entries.size()); - for (Int2LongMap.Entry kv : entries.int2LongEntrySet()) { - out.writeInt(kv.getIntKey()); - out.writeLong(kv.getLongValue()); + ObjectIterator<Int2LongMap.Entry> iter = + entries.int2LongEntrySet().fastIterator(); + while (iter.hasNext()) { + Int2LongMap.Entry entry = iter.next(); + out.writeInt(entry.getIntKey()); + out.writeLong(entry.getLongValue()); } }
