Author: pkosiorowski
Date: Mon Nov 14 14:39:29 2005
New Revision: 344254
URL: http://svn.apache.org/viewcvs?rev=344254&view=rev
Log:
NUTCH-99: ports are hardcoded or random. Patch by Stefan Groschupf.
Modified:
lucene/nutch/branches/mapred/conf/nutch-default.xml
lucene/nutch/branches/mapred/src/java/org/apache/nutch/mapred/JobTracker.java
lucene/nutch/branches/mapred/src/java/org/apache/nutch/mapred/TaskTracker.java
lucene/nutch/branches/mapred/src/java/org/apache/nutch/ndfs/DataNode.java
Modified: lucene/nutch/branches/mapred/conf/nutch-default.xml
URL:
http://svn.apache.org/viewcvs/lucene/nutch/branches/mapred/conf/nutch-default.xml?rev=344254&r1=344253&r2=344254&view=diff
==============================================================================
--- lucene/nutch/branches/mapred/conf/nutch-default.xml (original)
+++ lucene/nutch/branches/mapred/conf/nutch-default.xml Mon Nov 14 14:39:29 2005
@@ -349,6 +349,14 @@
</property>
<property>
+ <name>ndfs.datanode.port</name>
+ <value>50010</value>
+ <description>The port number that the ndfs datanode server uses as a
starting
+ point to look for a free port to listen on.
+</description>
+</property>
+
+<property>
<name>ndfs.name.dir</name>
<value>/tmp/nutch/ndfs/name</value>
<description>Determines where on the local filesystem the NDFS name node
@@ -372,6 +380,29 @@
<description>The host and port that the MapReduce job tracker runs
at. If "local", then jobs are run in-process as a single map
and reduce task.
+ </description>
+</property>
+
+<property>
+ <name>mapred.job.tracker.info.port</name>
+ <value>50030</value>
+ <description>The port that the MapReduce job tracker info webserver runs at.
+ </description>
+</property>
+
+<property>
+ <name>mapred.task.tracker.output.port</name>
+ <value>50040</value>
+ <description>The port number that the MapReduce task tracker output server
uses as a starting
+ point to look for a free port to listen on.
+ </description>
+</property>
+
+<property>
+ <name>mapred.task.tracker.report.port</name>
+ <value>50050</value>
+ <description>The port number that the MapReduce task tracker report server
uses as a starting
+ point to look for a free port to listen on.
</description>
</property>
Modified:
lucene/nutch/branches/mapred/src/java/org/apache/nutch/mapred/JobTracker.java
URL:
http://svn.apache.org/viewcvs/lucene/nutch/branches/mapred/src/java/org/apache/nutch/mapred/JobTracker.java?rev=344254&r1=344253&r2=344254&view=diff
==============================================================================
---
lucene/nutch/branches/mapred/src/java/org/apache/nutch/mapred/JobTracker.java
(original)
+++
lucene/nutch/branches/mapred/src/java/org/apache/nutch/mapred/JobTracker.java
Mon Nov 14 14:39:29 2005
@@ -237,7 +237,7 @@
LOG.info("Property '" + key + "' is " + val);
}
- this.infoPort = conf.getInt("mapred.job.tracker.info.port", 7845);
+ this.infoPort = conf.getInt("mapred.job.tracker.info.port", 50030);
this.infoServer = new JobTrackerInfoServer(this, infoPort);
this.infoServer.start();
Modified:
lucene/nutch/branches/mapred/src/java/org/apache/nutch/mapred/TaskTracker.java
URL:
http://svn.apache.org/viewcvs/lucene/nutch/branches/mapred/src/java/org/apache/nutch/mapred/TaskTracker.java?rev=344254&r1=344253&r2=344254&view=diff
==============================================================================
---
lucene/nutch/branches/mapred/src/java/org/apache/nutch/mapred/TaskTracker.java
(original)
+++
lucene/nutch/branches/mapred/src/java/org/apache/nutch/mapred/TaskTracker.java
Mon Nov 14 14:39:29 2005
@@ -67,17 +67,20 @@
NutchFileSystem fs = null;
static final String SUBDIR = "taskTracker";
+ private NutchConf fConf;
+
/**
* Start with the local machine name, and the default JobTracker
*/
- public TaskTracker() throws IOException {
- this(JobTracker.getAddress(NutchConf.get()));
+ public TaskTracker(NutchConf conf) throws IOException {
+ this(JobTracker.getAddress(conf), conf);
}
/**
* Start with the local machine name, and the addr of the target JobTracker
*/
- public TaskTracker(InetSocketAddress jobTrackAddr) throws IOException {
+ public TaskTracker(InetSocketAddress jobTrackAddr, NutchConf conf) throws
IOException {
+ this.fConf = conf;
this.jobTrackAddr = jobTrackAddr;
initialize();
}
@@ -97,17 +100,32 @@
this.tasks = new TreeMap();
this.runningTasks = new TreeMap();
- // generate port numbers
- this.taskReportPort = 32768+r.nextInt(32768);
- this.mapOutputPort = 32768+r.nextInt(32768);
+ // port numbers
+ this.taskReportPort =
this.fConf.getInt("mapred.task.tracker.report.port", 50050);
+ this.mapOutputPort =
this.fConf.getInt("mapred.task.tracker.output.port", 50040);
// RPC initialization
- this.taskReportServer =
- RPC.getServer(this, taskReportPort, MAX_CURRENT_TASKS, false);
- this.taskReportServer.start();
- this.mapOutputServer =
- RPC.getServer(this, mapOutputPort, MAX_CURRENT_TASKS, false);
- this.mapOutputServer.start();
+ while (true) {
+ try {
+ this.taskReportServer = RPC.getServer(this,
this.taskReportPort, MAX_CURRENT_TASKS, false);
+ this.taskReportServer.start();
+ break;
+ } catch (BindException e) {
+ LOG.info("Could not open report server at " +
this.taskReportPort + ", trying new port");
+ this.taskReportPort++;
+ }
+
+ }
+ while (true) {
+ try {
+ this.mapOutputServer = RPC.getServer(this, this.mapOutputPort,
MAX_CURRENT_TASKS, false);
+ this.mapOutputServer.start();
+ break;
+ } catch (BindException e) {
+ LOG.info("Could not open mapoutput server at " +
this.mapOutputPort + ", trying new port");
+ this.mapOutputPort++;
+ }
+ }
// Clear out temporary files that might be lying around
MapOutputFile.cleanupStorage();
@@ -629,7 +647,7 @@
System.exit(-1);
}
- TaskTracker tt = new TaskTracker();
+ TaskTracker tt = new TaskTracker(NutchConf.get());
tt.run();
}
}
Modified:
lucene/nutch/branches/mapred/src/java/org/apache/nutch/ndfs/DataNode.java
URL:
http://svn.apache.org/viewcvs/lucene/nutch/branches/mapred/src/java/org/apache/nutch/ndfs/DataNode.java?rev=344254&r1=344253&r2=344254&view=diff
==============================================================================
--- lucene/nutch/branches/mapred/src/java/org/apache/nutch/ndfs/DataNode.java
(original)
+++ lucene/nutch/branches/mapred/src/java/org/apache/nutch/ndfs/DataNode.java
Mon Nov 14 14:39:29 2005
@@ -66,6 +66,7 @@
Vector receivedBlockList = new Vector();
int xmitsInProgress = 0;
Daemon dataXceiveServer = null;
+ private NutchConf fConf;
/**
* Create given a configuration and a dataDir.
@@ -73,18 +74,18 @@
public DataNode(NutchConf conf, String datadir) throws IOException {
this(InetAddress.getLocalHost().getHostName(),
new File(datadir),
- createSocketAddr(conf.get("fs.default.name", "local")));
+ createSocketAddr(conf.get("fs.default.name", "local")), conf);
}
/**
* Needs a directory to find its data (and config info)
*/
- public DataNode(String machineName, File datadir, InetSocketAddress
nameNodeAddr) throws IOException {
+ public DataNode(String machineName, File datadir, InetSocketAddress
nameNodeAddr, NutchConf conf) throws IOException {
this.namenode = (DatanodeProtocol)
RPC.getProxy(DatanodeProtocol.class, nameNodeAddr);
this.data = new FSDataset(datadir);
ServerSocket ss = null;
- int tmpPort = 7000;
+ int tmpPort = conf.getInt("ndfs.datanode.port", 50010);
while (ss == null) {
try {
ss = new ServerSocket(tmpPort);
