Author: cutting
Date: Mon May 9 15:25:12 2005
New Revision: 169369
URL: http://svn.apache.org/viewcvs?rev=169369&view=rev
Log:
Add parameter to facilitate opening really big map files on machines with
little RAM.
Modified:
incubator/nutch/trunk/src/java/org/apache/nutch/io/MapFile.java
Modified: incubator/nutch/trunk/src/java/org/apache/nutch/io/MapFile.java
URL:
http://svn.apache.org/viewcvs/incubator/nutch/trunk/src/java/org/apache/nutch/io/MapFile.java?rev=169369&r1=169368&r2=169369&view=diff
==============================================================================
--- incubator/nutch/trunk/src/java/org/apache/nutch/io/MapFile.java (original)
+++ incubator/nutch/trunk/src/java/org/apache/nutch/io/MapFile.java Mon May 9
15:25:12 2005
@@ -44,6 +44,12 @@
/** The name of the data file. */
public static final String DATA_FILE_NAME = "data";
+ /** Number of index entries to skip between each entry. Zero by default.
+ * Setting this to values larger than zero can facilitate opening large map
+ * files using less memory. */
+ public static final int INDEX_SKIP =
+ NutchConf.get().getInt("io.map.index.skip", 0);
+
protected MapFile() {} // no public ctor
/** Writes a new map. */
@@ -203,6 +209,7 @@
this.keys = new WritableComparable[1024];
this.positions = new long[1024];
try {
+ int skip = INDEX_SKIP;
LongWritable position = new LongWritable();
WritableComparable lastKey = null;
while (true) {
@@ -216,6 +223,13 @@
throw new IOException("key out of order: "+k+" after "+lastKey);
lastKey = k;
+ if (skip > 0) {
+ skip--;
+ continue; // skip this entry
+ } else {
+ skip = INDEX_SKIP; // reset skip
+ }
+
if (count == keys.length) { // time to grow arrays
int newLength = (keys.length*3)/2;
WritableComparable[] newKeys = new WritableComparable[newLength];
-------------------------------------------------------
This SF.Net email is sponsored by: NEC IT Guy Games.
Get your fingers limbered up and give it your best shot. 4 great events, 4
opportunities to win big! Highest score wins.NEC IT Guy Games. Play to
win an NEC 61 plasma display. Visit http://www.necitguy.com/?r
_______________________________________________
Nutch-cvs mailing list
[email protected]
https://lists.sourceforge.net/lists/listinfo/nutch-cvs
