Author: cdouglas
Date: Thu Jan 8 01:14:46 2009
New Revision: 732655
URL: http://svn.apache.org/viewvc?rev=732655&view=rev
Log:
HADOOP-4947. Make Chukwa command parsing more forgiving of whitespace.
Contributed by Ari Rabkin.
Modified:
hadoop/core/trunk/CHANGES.txt
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/datacollection/adaptor/filetailer/FileTailingAdaptor.java
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/datacollection/agent/AgentControlSocketListener.java
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/datacollection/agent/ChukwaAgent.java
hadoop/core/trunk/src/contrib/chukwa/src/test/org/apache/hadoop/chukwa/datacollection/agent/TestAgent.java
Modified: hadoop/core/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/core/trunk/CHANGES.txt?rev=732655&r1=732654&r2=732655&view=diff
==============================================================================
--- hadoop/core/trunk/CHANGES.txt (original)
+++ hadoop/core/trunk/CHANGES.txt Thu Jan 8 01:14:46 2009
@@ -330,6 +330,9 @@
HADOOP-4980. Improve code layout of capacity scheduler to make it
easier to fix some blocker bugs. (Vivek Ratan via yhemanth)
+ HADOOP-4916. Make user/location of Chukwa installation configurable by an
+ external properties file. (Eric Yang via cdouglas)
+
OPTIMIZATIONS
HADOOP-3293. Fixes FileInputFormat to do provide locations for splits
@@ -555,12 +558,12 @@
HADOOP-4884. Make tool tip date format match standard HICC format. (Eric
Yang via cdouglas)
- HADOOP-4916. Make user/location of Chukwa installation configurable by an
- external properties file. (Eric Yang via cdouglas)
-
HADOOP-4925. Make Chukwa sender properties configurable. (Ari Rabkin via
cdouglas)
+ HADOOP-4947. Make Chukwa command parsing more forgiving of whitespace. (Ari
+ Rabkin via cdouglas)
+
Release 0.19.1 - Unreleased
IMPROVEMENTS
Modified:
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/datacollection/adaptor/filetailer/FileTailingAdaptor.java
URL:
http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/datacollection/adaptor/filetailer/FileTailingAdaptor.java?rev=732655&r1=732654&r2=732655&view=diff
==============================================================================
---
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/datacollection/adaptor/filetailer/FileTailingAdaptor.java
(original)
+++
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/datacollection/adaptor/filetailer/FileTailingAdaptor.java
Thu Jan 8 01:14:46 2009
@@ -29,6 +29,8 @@
import java.io.*;
import java.util.Timer;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
/**
* An adaptor that repeatedly tails a specified file, sending the new bytes.
@@ -81,11 +83,12 @@
this.type = type;
this.dest = dest;
this.attempts = 0;
-
- String[] words = params.split(" ");
- if(words.length > 1) {
- offsetOfFirstByte = Long.parseLong(words[0]);
- toWatch = new File(params.substring(words[0].length() + 1));
+
+ Pattern cmd = Pattern.compile("(\\d+)\\s+(.+)");
+ Matcher m = cmd.matcher(params);
+ if(m.matches()) {
+ offsetOfFirstByte = Long.parseLong(m.group(1));
+ toWatch = new File(m.group(2));
} else {
toWatch = new File(params);
}
Modified:
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/datacollection/agent/AgentControlSocketListener.java
URL:
http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/datacollection/agent/AgentControlSocketListener.java?rev=732655&r1=732654&r2=732655&view=diff
==============================================================================
---
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/datacollection/agent/AgentControlSocketListener.java
(original)
+++
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/datacollection/agent/AgentControlSocketListener.java
Thu Jan 8 01:14:46 2009
@@ -80,7 +80,7 @@
* @throws IOException
*/
public void processCommand(String cmd, PrintStream out) throws IOException
{
- String[] words = cmd.split(" ");
+ String[] words = cmd.split("\\s+");
if (log.isDebugEnabled())
{ log.debug("command from " +
connection.getRemoteSocketAddress() + ":"+ cmd);}
Modified:
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/datacollection/agent/ChukwaAgent.java
URL:
http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/datacollection/agent/ChukwaAgent.java?rev=732655&r1=732654&r2=732655&view=diff
==============================================================================
---
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/datacollection/agent/ChukwaAgent.java
(original)
+++
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/datacollection/agent/ChukwaAgent.java
Thu Jan 8 01:14:46 2009
@@ -30,6 +30,8 @@
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
import java.io.*;
/**
@@ -231,59 +233,45 @@
}
}
-
+ // words should contain (space delimited):
+ // 0) command ("add")
+ // 1) AdaptorClassname
+ // 2) dataType (e.g. "hadoop_log")
+ // 3) params <optional>
+ // (e.g. for files, this is filename,
+ // but can be arbitrarily many space
+ // delimited agent specific params )
+ // 4) offset
+ Pattern addCmdPattern =
Pattern.compile("add\\s+(\\S+)\\s+(\\S+)\\s+(.*\\S)?\\s*(\\d+)\\s*");
// FIXME: should handle bad lines here
public long processCommand(String cmd)
{
- String[] words = cmd.split(" ");
- if (words[0].equalsIgnoreCase("add"))
- {
- // words should contain (space delimited):
- // 0) command ("add")
- // 1) AdaptorClassname
- // 2) dataType (e.g. "hadoop_log")
- // 3) params <optional>
- // (e.g. for files, this is filename,
- // but can be arbitrarily many space
- // delimited agent specific params )
- // 4) offset
-
- long offset;
- try
- {
- offset = Long.parseLong(words[words.length - 1]);
- } catch (NumberFormatException e)
- {
+ Matcher m = addCmdPattern.matcher(cmd);
+ if (m.matches()) {
+ long offset; //check for obvious errors first
+ try {
+ offset = Long.parseLong(m.group(4));
+ } catch (NumberFormatException e) {
log.warn("malformed line " + cmd);
return -1L;
}
- String adaptorName = words[1];
+
+ String adaptorName = m.group(1);
+ String dataType = m.group(2);
+ String params = m.group(3);
+ if(params == null)
+ params = "";
Adaptor adaptor = AdaptorFactory.createAdaptor(adaptorName);
- if (adaptor == null)
- {
+ if (adaptor == null) {
log.warn("Error creating adaptor from adaptor name " + adaptorName);
return -1L;
}
- String dataType = words[2];
- String streamName = "";
- String params = "";
- if (words.length > 4)
- { // no argument
- int begParams = adaptorName.length() + dataType.length() + 6;
- // length("ADD x type ") = length(x) + 5, i.e. count letters & spaces
- params = cmd.substring(begParams, cmd.length()
- - words[words.length - 1].length() - 1);
- streamName = params.substring(params.indexOf(" ") + 1,
params.length());
- }
long adaptorID;
- synchronized (adaptorsByNumber)
- {
- for (Map.Entry<Long, Adaptor> a : adaptorsByNumber.entrySet())
- {
- if (streamName.intern() == a.getValue().getStreamName().intern())
- {
+ synchronized (adaptorsByNumber) {
+ for (Map.Entry<Long, Adaptor> a : adaptorsByNumber.entrySet()) {
+ if (params.intern() == a.getValue().getStreamName().intern()) {
log.warn(params + " already exist, skipping.");
return -1;
}
Modified:
hadoop/core/trunk/src/contrib/chukwa/src/test/org/apache/hadoop/chukwa/datacollection/agent/TestAgent.java
URL:
http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/src/test/org/apache/hadoop/chukwa/datacollection/agent/TestAgent.java?rev=732655&r1=732654&r2=732655&view=diff
==============================================================================
---
hadoop/core/trunk/src/contrib/chukwa/src/test/org/apache/hadoop/chukwa/datacollection/agent/TestAgent.java
(original)
+++
hadoop/core/trunk/src/contrib/chukwa/src/test/org/apache/hadoop/chukwa/datacollection/agent/TestAgent.java
Thu Jan 8 01:14:46 2009
@@ -65,7 +65,7 @@
ArrayList<Long> runningAdaptors = new ArrayList<Long>();
for(int i = 1; i < 7; ++i) {
- long l = agent.processCommand("add
org.apache.hadoop.chukwa.util.ConstRateAdaptor raw"+i+ " 2000"+i+" 0");
+ long l = agent.processCommand("add
org.apache.hadoop.chukwa.util.ConstRateAdaptor raw"+i+ " 2000"+i+" 0");
assertTrue(l != -1);
runningAdaptors.add(l);
}
