Author: asrabkin
Date: Wed Mar 4 20:27:23 2009
New Revision: 750140
URL: http://svn.apache.org/viewvc?rev=750140&view=rev
Log:
patch for HADOOP-5057
Improves test coverage for chukwa agent startup, in particular checkpoint
restore corner cases.
Patch by asrabkin.
Modified:
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/datacollection/agent/ChukwaAgent.java
hadoop/core/trunk/src/contrib/chukwa/src/test/org/apache/hadoop/chukwa/datacollection/adaptor/ChukwaTestAdaptor.java
hadoop/core/trunk/src/contrib/chukwa/src/test/org/apache/hadoop/chukwa/datacollection/agent/TestAgentConfig.java
Modified:
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/datacollection/agent/ChukwaAgent.java
URL:
http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/datacollection/agent/ChukwaAgent.java?rev=750140&r1=750139&r2=750140&view=diff
==============================================================================
---
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/datacollection/agent/ChukwaAgent.java
(original)
+++
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/datacollection/agent/ChukwaAgent.java
Wed Mar 4 20:27:23 2009
@@ -219,7 +219,6 @@
log.info("Config - tags: [" + tags + "]");
if (DO_CHECKPOINT_RESTORE) {
- needNewCheckpoint = true;
log.info("checkpoints are enabled, period is " + CHECKPOINT_INTERVAL_MS);
}
@@ -251,6 +250,9 @@
controlSock.start(); // this sets us up as a daemon
log.info("control socket started on port " + controlSock.portno);
+
+ //shouldn't start checkpointing until we're finishing launching
+ //adaptors on boot
if (CHECKPOINT_INTERVAL_MS > 0 && checkpointDir!= null) {
checkpointer = new Timer();
checkpointer.schedule(new CheckpointTask(), 0, CHECKPOINT_INTERVAL_MS);
@@ -315,6 +317,7 @@
adaptorID = ++lastAdaptorNumber;
adaptorsByNumber.put(adaptorID, adaptor);
adaptorPositions.put(adaptor, new Offset(offset, adaptorID));
+ needNewCheckpoint = true;
try
{
adaptor.start(adaptorID, dataType, params, offset, DataFactory
@@ -330,7 +333,7 @@
}
} else
if(cmd.length() > 0)
- log.warn("only 'add' command supported in config files");
+ log.warn("only 'add' command supported in config files; cmd was: "+
cmd);
//no warning for blank line
return -1;
@@ -361,7 +364,7 @@
});
if (checkpointNames == null) {
- log.error("Unable to list directories in checkpoint dir");
+ log.error("Unable to list files in checkpoint dir");
return false;
}
if (checkpointNames.length == 0)
@@ -576,7 +579,8 @@
if (checkpointer != null) {
checkpointer.cancel();
try {
- writeCheckpoint(); // write a last checkpoint here, before stopping
+ if(needNewCheckpoint)
+ writeCheckpoint(); // write a last checkpoint here, before stopping
} catch (IOException e) {
}
}
Modified:
hadoop/core/trunk/src/contrib/chukwa/src/test/org/apache/hadoop/chukwa/datacollection/adaptor/ChukwaTestAdaptor.java
URL:
http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/src/test/org/apache/hadoop/chukwa/datacollection/adaptor/ChukwaTestAdaptor.java?rev=750140&r1=750139&r2=750140&view=diff
==============================================================================
---
hadoop/core/trunk/src/contrib/chukwa/src/test/org/apache/hadoop/chukwa/datacollection/adaptor/ChukwaTestAdaptor.java
(original)
+++
hadoop/core/trunk/src/contrib/chukwa/src/test/org/apache/hadoop/chukwa/datacollection/adaptor/ChukwaTestAdaptor.java
Wed Mar 4 20:27:23 2009
@@ -31,13 +31,13 @@
@Override
public String getCurrentStatus() throws AdaptorException {
// TODO Auto-generated method stub
- return null;
+ return type+ " "+ params + " "+ startOffset;
}
@Override
public String getStreamName() {
// TODO Auto-generated method stub
- return null;
+ return "";
}
@Override
Modified:
hadoop/core/trunk/src/contrib/chukwa/src/test/org/apache/hadoop/chukwa/datacollection/agent/TestAgentConfig.java
URL:
http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/src/test/org/apache/hadoop/chukwa/datacollection/agent/TestAgentConfig.java?rev=750140&r1=750139&r2=750140&view=diff
==============================================================================
---
hadoop/core/trunk/src/contrib/chukwa/src/test/org/apache/hadoop/chukwa/datacollection/agent/TestAgentConfig.java
(original)
+++
hadoop/core/trunk/src/contrib/chukwa/src/test/org/apache/hadoop/chukwa/datacollection/agent/TestAgentConfig.java
Wed Mar 4 20:27:23 2009
@@ -73,4 +73,60 @@
fail(e.toString());
}
}
+
+
+ public void testNoCheckpoints() {
+ try {
+ String tmpdir = System.getProperty("test.build.data", "/tmp");
+ File NONCE_DIR = new File(tmpdir,"/test_chukwa_checkpoints");
+ if(NONCE_DIR.exists()) {
+ for(File f: NONCE_DIR.listFiles())
+ f.delete();
+ NONCE_DIR.delete();
+ }
+// assertFalse(NONCE_DIR.exists());
+ Configuration conf = new Configuration();
+ conf.set("chukwaAgent.checkpoint.dir", NONCE_DIR.getAbsolutePath());
+ conf.setBoolean("chukwaAgent.checkpoint.enabled", true);
+ conf.setInt("chukwaAgent.agent.control.port", 0);
+
+ System.out.println("\n\n===checkpoints enabled, dir does not exist:");
+ ChukwaAgent agent = new ChukwaAgent(conf);
+ assertEquals(0, agent.getAdaptorList().size());
+ agent.shutdown();
+ assertTrue(NONCE_DIR.exists());
+ for(File f: NONCE_DIR.listFiles())
+ f.delete();
+
+ System.out.println("\n\n===checkpoints enabled, dir exists but is
empty:");
+ agent = new ChukwaAgent(conf);
+ assertEquals(0, agent.getAdaptorList().size());
+ agent.shutdown();
+ for(File f: NONCE_DIR.listFiles())
+ f.delete();
+
+ System.out.println("\n\n===checkpoints enabled, dir exists with
zero-length file:");
+ (new File(NONCE_DIR, "chukwa_checkpoint_0")).createNewFile();
+ agent = new ChukwaAgent(conf);
+ assertEquals(0, agent.getAdaptorList().size());
+ agent.processCommand("ADD
org.apache.hadoop.chukwa.datacollection.adaptor.ChukwaTestAdaptor testdata 0");
+ agent.shutdown();
+ assertTrue(new File(NONCE_DIR, "chukwa_checkpoint_1").exists());
+
+ System.out.println("\n\n===checkpoints enabled, dir exists with valid
checkpoint");
+ agent = new ChukwaAgent(conf);
+ assertEquals(1, agent.getAdaptorList().size());
+ agent.shutdown();
+ //checkpoint # increments by one on boot and reload
+ assertTrue(new File(NONCE_DIR, "chukwa_checkpoint_2").exists());
+
+
+ }
+ catch(Exception e) {
+ fail(e.toString());
+ }
+ }
+
+
+
}
