I noticed that the splitter processing does a large number of small read
& writes. The speed can be doubled by using buffered IO as per the
attached patch.
Before:
$ time java -Xmx1500m -jar splitter/dist/splitter.jar --max-nodes=1000000
--cache=cache /store/planet/great_britain-20091114.osm.bz2
...
Wrote 11,101,332 nodes, 1,485,442 ways, 54,180 relations
Time finished: Sat Nov 14 13:43:40 GMT 2009
Total time taken: 675s
real 11m15.561s
user 6m17.064s
sys 4m19.885s
After:
Wrote 11,101,332 nodes, 1,485,442 ways, 54,180 relations
Time finished: Sat Nov 14 14:17:37 GMT 2009
Total time taken: 305s
real 5m5.343s
user 4m42.738s
sys 0m8.555s
I did remember to delete the cache between the two runs.
Jon
Index: src/uk/me/parabola/splitter/Utils.java
===================================================================
--- src/uk/me/parabola/splitter/Utils.java (revision 97)
+++ src/uk/me/parabola/splitter/Utils.java (working copy)
@@ -18,6 +18,7 @@
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.Reader;
+import java.io.BufferedInputStream;
import java.nio.charset.Charset;
import java.text.NumberFormat;
import java.util.zip.GZIPInputStream;
@@ -91,7 +92,7 @@
}
} else if (name.endsWith(".bz2")) {
try {
- is.read(); is.read(); is = new CBZip2InputStream(is);
+ is.read(); is.read(); is = new CBZip2InputStream(new BufferedInputStream(is,10240));
} catch (IOException e) {
throw new IOException( "Could not read " + name + " as a bz2 compressed file", e);
}
Index: src/uk/me/parabola/splitter/disk/NodeStoreWriter.java
===================================================================
--- src/uk/me/parabola/splitter/disk/NodeStoreWriter.java (revision 97)
+++ src/uk/me/parabola/splitter/disk/NodeStoreWriter.java (working copy)
@@ -12,6 +12,7 @@
*/
package uk.me.parabola.splitter.disk;
+import java.io.BufferedOutputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStream;
@@ -22,7 +23,7 @@
public class NodeStoreWriter extends AbstractStoreWriter {
public NodeStoreWriter(String filename) throws IOException {
- this(new FileOutputStream(filename), new KeyLookupWriter(filename + ".keys"));
+ this(new BufferedOutputStream(new FileOutputStream(filename),10240), new KeyLookupWriter(filename + ".keys"));
}
public NodeStoreWriter(OutputStream out, KeyLookupWriter keys) {
Index: src/uk/me/parabola/splitter/disk/WayStoreWriter.java
===================================================================
--- src/uk/me/parabola/splitter/disk/WayStoreWriter.java (revision 97)
+++ src/uk/me/parabola/splitter/disk/WayStoreWriter.java (working copy)
@@ -12,6 +12,7 @@
*/
package uk.me.parabola.splitter.disk;
+import java.io.BufferedOutputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStream;
@@ -22,7 +23,7 @@
public class WayStoreWriter extends AbstractStoreWriter {
public WayStoreWriter(String filename) throws IOException {
- this(new FileOutputStream(filename), new KeyLookupWriter(filename + ".keys"));
+ this(new BufferedOutputStream(new FileOutputStream(filename),10240), new KeyLookupWriter(filename + ".keys"));
}
public WayStoreWriter(OutputStream out, KeyLookupWriter keys) {
Index: src/uk/me/parabola/splitter/disk/RelationStoreWriter.java
===================================================================
--- src/uk/me/parabola/splitter/disk/RelationStoreWriter.java (revision 97)
+++ src/uk/me/parabola/splitter/disk/RelationStoreWriter.java (working copy)
@@ -12,6 +12,7 @@
*/
package uk.me.parabola.splitter.disk;
+import java.io.BufferedOutputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStream;
@@ -23,7 +24,7 @@
private final KeyLookupWriter roles;
public RelationStoreWriter(String filename) throws IOException {
- this(new FileOutputStream(filename), new KeyLookupWriter(filename + ".keys"), new KeyLookupWriter(filename + ".roles"));
+ this(new BufferedOutputStream(new FileOutputStream(filename),10240), new KeyLookupWriter(filename + ".keys"), new KeyLookupWriter(filename + ".roles"));
}
public RelationStoreWriter(OutputStream out, KeyLookupWriter keys, KeyLookupWriter roles) {
_______________________________________________
mkgmap-dev mailing list
[email protected]
http://www.mkgmap.org.uk/mailman/listinfo/mkgmap-dev
