Update of /cvsroot/nutch/nutch/src/java/net/nutch/fs
In directory sc8-pr-cvs1.sourceforge.net:/tmp/cvs-serv9735/java/net/nutch/fs
Modified Files:
FSNamesystem.java NDFS.java TestClient.java
Log Message:
NDFS datanode size is now configurable, defaults to 100Gb.
Fix problem with NDFS when a client dies, but its lease does
not expire correctly. Lease queue was not managed correctly.
Index: TestClient.java
===================================================================
RCS file: /cvsroot/nutch/nutch/src/java/net/nutch/fs/TestClient.java,v
retrieving revision 1.3
retrieving revision 1.4
diff -C2 -d -r1.3 -r1.4
*** TestClient.java 20 Aug 2004 20:36:11 -0000 1.3
--- TestClient.java 29 Nov 2004 05:50:33 -0000 1.4
***************
*** 193,221 ****
int i = 0;
NutchFileSystem nfs = NutchFileSystem.parseArgs(argv, i);
! TestClient tc = new TestClient(nfs);
! String cmd = argv[i++];
! if ("-put".equals(cmd)) {
! tc.put(new File(argv[i++]), argv[i++]);
! } else if ("-get".equals(cmd)) {
! tc.get(argv[i++], new File(argv[i++]));
! } else if ("-ls".equals(cmd)) {
! tc.ls(argv[i++]);
! } else if ("-mv".equals(cmd)) {
! tc.rename(argv[i++], argv[i++]);
! } else if ("-cp".equals(cmd)) {
! tc.copy(argv[i++], argv[i++]);
! } else if ("-rm".equals(cmd)) {
! tc.delete(argv[i++]);
! } else if ("-ls".equals(cmd)) {
! tc.ls(argv[i++]);
! } else if ("-du".equals(cmd)) {
! tc.du(argv[i++]);
! } else if ("-mkdir".equals(cmd)) {
! tc.mkdir(argv[i++]);
! } else if ("-report".equals(cmd)) {
! tc.report();
}
- System.exit(0);
}
}
--- 193,225 ----
int i = 0;
NutchFileSystem nfs = NutchFileSystem.parseArgs(argv, i);
! try {
! TestClient tc = new TestClient(nfs);
! String cmd = argv[i++];
! if ("-put".equals(cmd)) {
! tc.put(new File(argv[i++]), argv[i++]);
! } else if ("-get".equals(cmd)) {
! tc.get(argv[i++], new File(argv[i++]));
! } else if ("-ls".equals(cmd)) {
! tc.ls(argv[i++]);
! } else if ("-mv".equals(cmd)) {
! tc.rename(argv[i++], argv[i++]);
! } else if ("-cp".equals(cmd)) {
! tc.copy(argv[i++], argv[i++]);
! } else if ("-rm".equals(cmd)) {
! tc.delete(argv[i++]);
! } else if ("-ls".equals(cmd)) {
! tc.ls(argv[i++]);
! } else if ("-du".equals(cmd)) {
! tc.du(argv[i++]);
! } else if ("-mkdir".equals(cmd)) {
! tc.mkdir(argv[i++]);
! } else if ("-report".equals(cmd)) {
! tc.report();
! }
! System.exit(0);
! } finally {
! nfs.close();
}
}
}
Index: NDFS.java
===================================================================
RCS file: /cvsroot/nutch/nutch/src/java/net/nutch/fs/NDFS.java,v
retrieving revision 1.5
retrieving revision 1.6
diff -C2 -d -r1.5 -r1.6
*** NDFS.java 20 Aug 2004 22:42:11 -0000 1.5
--- NDFS.java 29 Nov 2004 05:50:33 -0000 1.6
***************
*** 20,23 ****
--- 20,24 ----
public static final Logger LOG =
LogFormatter.getLogger("net.nutch.fs.NDFS");
private static final long GIGABYTE = 1024 * 1024 * 1024;
+ private static long numGigs = NutchConf.getLong("ndfs.datanode.maxgigs",
100);
//
***************
*** 381,386 ****
public DataNode(String machineName, File dir, InetSocketAddress
nameNodeAddr) throws IOException {
super(FSResults.class);
! // REMIND - mjc - load from config file
! long capacity = 60 * GIGABYTE;
this.data = new FSDataset(dir, capacity);
this.nameNodeAddr = nameNodeAddr;
--- 382,386 ----
public DataNode(String machineName, File dir, InetSocketAddress
nameNodeAddr) throws IOException {
super(FSResults.class);
! long capacity = numGigs * GIGABYTE;
this.data = new FSDataset(dir, capacity);
this.nameNodeAddr = nameNodeAddr;
Index: FSNamesystem.java
===================================================================
RCS file: /cvsroot/nutch/nutch/src/java/net/nutch/fs/FSNamesystem.java,v
retrieving revision 1.5
retrieving revision 1.6
diff -C2 -d -r1.5 -r1.6
*** FSNamesystem.java 1 Sep 2004 17:58:21 -0000 1.5
--- FSNamesystem.java 29 Nov 2004 05:50:33 -0000 1.6
***************
*** 105,123 ****
//
TreeMap leases = new TreeMap();
! TreeSet sortedLeases = new TreeSet(new Comparator() {
! public int compare(Object o1, Object o2) {
! Lease l1 = (Lease) o1;
! Lease l2 = (Lease) o2;
! long lu1 = l1.lastUpdate;
! long lu2 = l2.lastUpdate;
! if (lu1 < lu2) {
! return -1;
! } else if (lu1 > lu2) {
! return 1;
! } else {
! return l1.holder.compareTo(l2.holder);
! }
! }
! });
//
--- 105,109 ----
//
TreeMap leases = new TreeMap();
! TreeSet sortedLeases = new TreeSet();
//
***************
*** 231,235 ****
--- 217,223 ----
sortedLeases.add(lease);
} else {
+ sortedLeases.remove(lease);
lease.renew();
+ sortedLeases.add(lease);
}
lease.startedCreate(src);
***************
*** 490,494 ****
* expire, all the corresponding locks can be released.
*************************************************************/
! class Lease {
public UTF8 holder;
public long lastUpdate;
--- 478,482 ----
* expire, all the corresponding locks can be released.
*************************************************************/
! class Lease implements Comparable {
public UTF8 holder;
public long lastUpdate;
***************
*** 497,500 ****
--- 485,489 ----
public Lease(UTF8 holder) {
+ LOG.info("New lease, holder " + holder);
this.holder = holder;
renew();
***************
*** 534,537 ****
--- 523,548 ----
creates.clear();
}
+
+ /**
+ */
+ public String toString() {
+ return "[Lease. Holder: " + holder.toString() + ", heldlocks: "
+ locks.size() + ", pendingcreates: " + creates.size() + "]";
+ }
+
+ /**
+ */
+ public int compareTo(Object o) {
+ Lease l1 = (Lease) this;
+ Lease l2 = (Lease) o;
+ long lu1 = l1.lastUpdate;
+ long lu2 = l2.lastUpdate;
+ if (lu1 < lu2) {
+ return -1;
+ } else if (lu1 > lu2) {
+ return 1;
+ } else {
+ return l1.holder.compareTo(l2.holder);
+ }
+ }
}
/******************************************************
***************
*** 550,554 ****
top.releaseLocks();
leases.remove(top.holder);
! sortedLeases.remove(top);
} else {
break;
--- 561,568 ----
top.releaseLocks();
leases.remove(top.holder);
! LOG.info("Removing lease " + top + ", leases
remaining: " + sortedLeases.size());
! if (!sortedLeases.remove(top)) {
! LOG.info("Unknown failure trying to
remove " + top + " from lease set.");
! }
} else {
break;
***************
*** 578,582 ****
--- 592,598 ----
sortedLeases.add(lease);
} else {
+ sortedLeases.remove(lease);
lease.renew();
+ sortedLeases.add(lease);
}
lease.obtained(src);
***************
*** 605,612 ****
return result;
}
! private synchronized int internalReleaseLock(UTF8 src, UTF8 holder) {
return dir.releaseLock(src, holder);
}
! private synchronized void internalReleaseCreates(TreeSet creates) {
for (Iterator it = creates.iterator(); it.hasNext(); ) {
UTF8 src = (UTF8) it.next();
--- 621,628 ----
return result;
}
! private int internalReleaseLock(UTF8 src, UTF8 holder) {
return dir.releaseLock(src, holder);
}
! private void internalReleaseCreates(TreeSet creates) {
for (Iterator it = creates.iterator(); it.hasNext(); ) {
UTF8 src = (UTF8) it.next();
***************
*** 629,632 ****
--- 645,649 ----
lease.renew();
sortedLeases.add(lease);
+ LOG.info("Renewed lease " + lease);
}
}
-------------------------------------------------------
SF email is sponsored by - The IT Product Guide
Read honest & candid reviews on hundreds of IT Products from real users.
Discover which products truly live up to the hype. Start reading now.
http://productguide.itmanagersjournal.com/
_______________________________________________
Nutch-cvs mailing list
[EMAIL PROTECTED]
https://lists.sourceforge.net/lists/listinfo/nutch-cvs
