Repository: chukwa Updated Branches: refs/heads/master e29381f07 -> 7f662e8c6
http://git-wip-us.apache.org/repos/asf/chukwa/blob/6012e14f/src/test/java/org/apache/hadoop/chukwa/datacollection/adaptor/filetailer/TestFileTailingAdaptors.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/hadoop/chukwa/datacollection/adaptor/filetailer/TestFileTailingAdaptors.java b/src/test/java/org/apache/hadoop/chukwa/datacollection/adaptor/filetailer/TestFileTailingAdaptors.java index a71790d..3bda707 100644 --- a/src/test/java/org/apache/hadoop/chukwa/datacollection/adaptor/filetailer/TestFileTailingAdaptors.java +++ b/src/test/java/org/apache/hadoop/chukwa/datacollection/adaptor/filetailer/TestFileTailingAdaptors.java @@ -17,7 +17,6 @@ */ package org.apache.hadoop.chukwa.datacollection.adaptor.filetailer; - import java.io.*; import junit.framework.TestCase; @@ -36,7 +35,7 @@ public class TestFileTailingAdaptors extends TestCase { ChunkCatcherConnector chunks; Configuration conf = new Configuration(); File baseDir, testFile; - + public TestFileTailingAdaptors() throws IOException { chunks = new ChunkCatcherConnector(); chunks.start(); @@ -46,13 +45,14 @@ public class TestFileTailingAdaptors extends TestCase { conf.setInt("chukwaAgent.adaptor.context.switch.time", 100); conf.set("chukwaAgent.control.port", "0"); - testFile = makeTestFile("chukwaCrSepTest", 80,baseDir); + testFile = makeTestFile("chukwaCrSepTest", 80, baseDir); } public void testCrSepAdaptor() throws IOException, InterruptedException, ChukwaAgent.AlreadyRunningException { - ChukwaAgent agent = new ChukwaAgent(conf); + ChukwaAgent agent = ChukwaAgent.getAgent(conf); + agent.start(); // Remove any adaptor left over from previous run // sleep for some time to make sure we don't get chunk from existing streams @@ -84,81 +84,90 @@ public class TestFileTailingAdaptors extends TestCase { agent.shutdown(); Thread.sleep(2000); } - + public void testRepeatedlyOnBigFile() throws IOException, - ChukwaAgent.AlreadyRunningException, InterruptedException { - int tests = 10; //SHOULD SET HIGHER AND WATCH WITH lsof to find leaks + ChukwaAgent.AlreadyRunningException, InterruptedException { + int tests = 10; // SHOULD SET HIGHER AND WATCH WITH lsof to find leaks - ChukwaAgent agent = new ChukwaAgent(conf); - for(int i=0; i < tests; ++i) { - if(i % 100 == 0) + ChukwaAgent agent = ChukwaAgent.getAgent(conf); + agent.start(); + for (int i = 0; i < tests; ++i) { + if (i % 100 == 0) System.out.println("buzzed " + i + " times"); - + assertEquals(0, agent.adaptorCount()); - agent.processAddCommand("add adaptor_test = filetailer.FileTailingAdaptor raw " +testFile.getCanonicalPath() + " 0"); + agent + .processAddCommand("add adaptor_test = filetailer.FileTailingAdaptor raw " + + testFile.getCanonicalPath() + " 0"); assertEquals(1, agent.adaptorCount()); Chunk c = chunks.waitForAChunk(); String dat = new String(c.getData()); assertTrue(dat.startsWith("0 abcdefghijklmnopqrstuvwxyz")); assertTrue(dat.endsWith("9 abcdefghijklmnopqrstuvwxyz\n")); assertTrue(c.getDataType().equals("raw")); - if(agent.adaptorCount() > 0) + if (agent.adaptorCount() > 0) agent.stopAdaptor("adaptor_test", false); } agent.shutdown(); } - - public void testOffsetInAdaptorName() throws IOException, ChukwaAgent.AlreadyRunningException, - InterruptedException{ - File testFile = makeTestFile("foo", 120,baseDir); - ChukwaAgent agent = new ChukwaAgent(conf); + public void testOffsetInAdaptorName() throws IOException, + ChukwaAgent.AlreadyRunningException, InterruptedException { + File testFile = makeTestFile("foo", 120, baseDir); + ChukwaAgent agent = ChukwaAgent.getAgent(conf); + agent.start(); assertEquals(0, agent.adaptorCount()); - agent.processAddCommand("add test = filetailer.FileTailingAdaptor raw " +testFile.getCanonicalPath() + " 0"); + agent.processAddCommand("add test = filetailer.FileTailingAdaptor raw " + + testFile.getCanonicalPath() + " 0"); assertEquals(1, agent.adaptorCount()); Thread.sleep(2000); - agent.processAddCommand("add test = filetailer.FileTailingAdaptor raw " +testFile.getCanonicalPath() + " 0"); + agent.processAddCommand("add test = filetailer.FileTailingAdaptor raw " + + testFile.getCanonicalPath() + " 0"); assertEquals(1, agent.adaptorCount()); chunks.clear(); agent.shutdown(); } - + /** - * Test that LWFTAdaptor updates lastSlurpTime so that FileTailingAdaptor - * does not trigger an infinite loop and that slurp() is not called by + * Test that LWFTAdaptor updates lastSlurpTime so that FileTailingAdaptor does + * not trigger an infinite loop and that slurp() is not called by * FileTailingAdaptor if file is not updated (see CHUKWA-668) + * * @throws IOException * @throws ChukwaAgent.AlreadyRunningException * @throws InterruptedException */ - public void testSlurpTimeUpdated() throws IOException, ChukwaAgent.AlreadyRunningException, - InterruptedException{ - ChukwaAgent agent = new ChukwaAgent(conf); - File testFile = makeTestFile("fooSlurp", 0,baseDir); - long startTime = System.currentTimeMillis(); - String adaptorId = agent.processAddCommand("add adaptor_test =" + - "filetailer.FileTailingAdaptor slurp " +testFile.getCanonicalPath() + " 0"); - FileTailingAdaptor fta = (FileTailingAdaptor)agent.getAdaptor( adaptorId); - Thread.sleep(500); - long initializedSlurpTimeValue = fta.lastSlurpTime; - assertTrue( initializedSlurpTimeValue > startTime); // initialized to current time - - makeTestFile("fooSlurp", 2,baseDir); - Chunk c = chunks.waitForAChunk(); - - Thread.sleep(2000); - // lastSlurpTime has been updated because a slurp was done - long secondSlurpTimeValue = fta.lastSlurpTime; - assertTrue( secondSlurpTimeValue > initializedSlurpTimeValue); - assertEquals( fta.fileReadOffset, c.getData().length); - assertEquals( fta.fileReadOffset, fta.reader.length()); - - Thread.sleep(2000); - // ensure we don't try to slurp if file is not updated - assertEquals( fta.lastSlurpTime, secondSlurpTimeValue); - - if(agent.adaptorCount() > 0) - agent.stopAdaptor("adaptor_test", false); - agent.shutdown(); + public void testSlurpTimeUpdated() throws IOException, + ChukwaAgent.AlreadyRunningException, InterruptedException { + ChukwaAgent agent = ChukwaAgent.getAgent(conf); + agent.start(); + File testFile = makeTestFile("fooSlurp", 0, baseDir); + long startTime = System.currentTimeMillis(); + String adaptorId = agent.processAddCommand("add adaptor_test =" + + "filetailer.FileTailingAdaptor slurp " + testFile.getCanonicalPath() + + " 0"); + FileTailingAdaptor fta = (FileTailingAdaptor) agent.getAdaptor(adaptorId); + Thread.sleep(500); + long initializedSlurpTimeValue = fta.lastSlurpTime; + assertTrue(initializedSlurpTimeValue > startTime); // initialized to current + // time + + makeTestFile("fooSlurp", 2, baseDir); + Chunk c = chunks.waitForAChunk(); + + Thread.sleep(2000); + // lastSlurpTime has been updated because a slurp was done + long secondSlurpTimeValue = fta.lastSlurpTime; + assertTrue(secondSlurpTimeValue > initializedSlurpTimeValue); + assertEquals(fta.fileReadOffset, c.getData().length); + assertEquals(fta.fileReadOffset, fta.reader.length()); + + Thread.sleep(2000); + // ensure we don't try to slurp if file is not updated + assertEquals(fta.lastSlurpTime, secondSlurpTimeValue); + + if (agent.adaptorCount() > 0) + agent.stopAdaptor("adaptor_test", false); + agent.shutdown(); } } http://git-wip-us.apache.org/repos/asf/chukwa/blob/6012e14f/src/test/java/org/apache/hadoop/chukwa/datacollection/adaptor/filetailer/TestLogRotate.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/hadoop/chukwa/datacollection/adaptor/filetailer/TestLogRotate.java b/src/test/java/org/apache/hadoop/chukwa/datacollection/adaptor/filetailer/TestLogRotate.java index ba0e15d..6f9e2d3 100644 --- a/src/test/java/org/apache/hadoop/chukwa/datacollection/adaptor/filetailer/TestLogRotate.java +++ b/src/test/java/org/apache/hadoop/chukwa/datacollection/adaptor/filetailer/TestLogRotate.java @@ -41,6 +41,7 @@ public class TestLogRotate extends TestCase { public void testLogRotate() throws IOException, InterruptedException, ChukwaAgent.AlreadyRunningException { ChukwaAgent agent = ChukwaAgent.getAgent(); + agent.start(); // Remove any adaptor left over from previous run ChukwaConfiguration cc = new ChukwaConfiguration(); int portno = cc.getInt("chukwaAgent.control.port", 9093); http://git-wip-us.apache.org/repos/asf/chukwa/blob/6012e14f/src/test/java/org/apache/hadoop/chukwa/datacollection/adaptor/filetailer/TestRCheckAdaptor.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/hadoop/chukwa/datacollection/adaptor/filetailer/TestRCheckAdaptor.java b/src/test/java/org/apache/hadoop/chukwa/datacollection/adaptor/filetailer/TestRCheckAdaptor.java index 7c5c2d3..f4ec73f 100644 --- a/src/test/java/org/apache/hadoop/chukwa/datacollection/adaptor/filetailer/TestRCheckAdaptor.java +++ b/src/test/java/org/apache/hadoop/chukwa/datacollection/adaptor/filetailer/TestRCheckAdaptor.java @@ -54,7 +54,8 @@ public class TestRCheckAdaptor extends TestCase implements ChunkReceiver { Configuration conf = new ChukwaConfiguration(); conf.set("", "org.apache.hadoop.chukwa.datacollection.connector.ChunkCatcherConnector"); try { - agent = new ChukwaAgent(conf); + agent = ChukwaAgent.getAgent(conf); + agent.start(); Thread.sleep(2000); Map<String, String> adaptorList = agent.getAdaptorList(); for(String id : adaptorList.keySet()) { http://git-wip-us.apache.org/repos/asf/chukwa/blob/6012e14f/src/test/java/org/apache/hadoop/chukwa/datacollection/adaptor/filetailer/TestRawAdaptor.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/hadoop/chukwa/datacollection/adaptor/filetailer/TestRawAdaptor.java b/src/test/java/org/apache/hadoop/chukwa/datacollection/adaptor/filetailer/TestRawAdaptor.java index 935ddd1..d2334d8 100644 --- a/src/test/java/org/apache/hadoop/chukwa/datacollection/adaptor/filetailer/TestRawAdaptor.java +++ b/src/test/java/org/apache/hadoop/chukwa/datacollection/adaptor/filetailer/TestRawAdaptor.java @@ -64,7 +64,8 @@ public class TestRawAdaptor extends TestCase { Configuration conf = new Configuration(); conf.set("chukwaAgent.control.port", "0"); conf.setInt("chukwaAgent.adaptor.context.switch.time", 100); - ChukwaAgent agent = new ChukwaAgent(conf); + ChukwaAgent agent = ChukwaAgent.getAgent(conf); + agent.start(); File testFile = makeTestFile("chukwaRawTest", 80, new File(System.getProperty("test.build.data", "/tmp"))); http://git-wip-us.apache.org/repos/asf/chukwa/blob/6012e14f/src/test/java/org/apache/hadoop/chukwa/datacollection/adaptor/filetailer/TestStartAtOffset.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/hadoop/chukwa/datacollection/adaptor/filetailer/TestStartAtOffset.java b/src/test/java/org/apache/hadoop/chukwa/datacollection/adaptor/filetailer/TestStartAtOffset.java index a2ee740..363b1ce 100644 --- a/src/test/java/org/apache/hadoop/chukwa/datacollection/adaptor/filetailer/TestStartAtOffset.java +++ b/src/test/java/org/apache/hadoop/chukwa/datacollection/adaptor/filetailer/TestStartAtOffset.java @@ -46,7 +46,8 @@ public class TestStartAtOffset extends TestCase { Configuration conf = new Configuration(); conf.set("chukwaAgent.control.port", "0"); conf.setInt("chukwaAgent.adaptor.context.switch.time", 100); - ChukwaAgent agent = new ChukwaAgent(conf); + ChukwaAgent agent = ChukwaAgent.getAgent(conf); + agent.start(); File testFile = makeTestFile(); int startOffset = 0; // skip first line String adaptorId = agent @@ -81,7 +82,8 @@ public class TestStartAtOffset extends TestCase { Configuration conf = new Configuration(); conf.set("chukwaAgent.control.port", "0"); conf.setInt("chukwaAgent.adaptor.context.switch.time", 100); - ChukwaAgent agent = new ChukwaAgent(conf); + ChukwaAgent agent = ChukwaAgent.getAgent(conf); + agent.start(); File testFile = makeTestFile(); int startOffset = 0; String adaptorId = agent http://git-wip-us.apache.org/repos/asf/chukwa/blob/6012e14f/src/test/java/org/apache/hadoop/chukwa/datacollection/agent/TestAgent.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/hadoop/chukwa/datacollection/agent/TestAgent.java b/src/test/java/org/apache/hadoop/chukwa/datacollection/agent/TestAgent.java index 07c160c..dd5b702 100644 --- a/src/test/java/org/apache/hadoop/chukwa/datacollection/agent/TestAgent.java +++ b/src/test/java/org/apache/hadoop/chukwa/datacollection/agent/TestAgent.java @@ -34,7 +34,8 @@ public class TestAgent extends TestCase { try { Configuration conf = new Configuration(); conf.setInt("chukwaAgent.control.port", 0); - ChukwaAgent agent = new ChukwaAgent(conf); + ChukwaAgent agent = ChukwaAgent.getAgent(conf); + agent.start(); ConsoleOutConnector conn = new ConsoleOutConnector(agent, true); conn.start(); @@ -63,7 +64,8 @@ public class TestAgent extends TestCase { try { Configuration conf = new Configuration(); conf.setInt("chukwaAgent.control.port", 0); - ChukwaAgent agent = new ChukwaAgent(conf); + ChukwaAgent agent = ChukwaAgent.getAgent(conf); + agent.start(); ConsoleOutConnector conn = new ConsoleOutConnector(agent, true); conn.start(); int count = agent.adaptorCount(); http://git-wip-us.apache.org/repos/asf/chukwa/blob/6012e14f/src/test/java/org/apache/hadoop/chukwa/datacollection/agent/TestAgentConfig.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/hadoop/chukwa/datacollection/agent/TestAgentConfig.java b/src/test/java/org/apache/hadoop/chukwa/datacollection/agent/TestAgentConfig.java index 8f4eb73..c960b4d 100644 --- a/src/test/java/org/apache/hadoop/chukwa/datacollection/agent/TestAgentConfig.java +++ b/src/test/java/org/apache/hadoop/chukwa/datacollection/agent/TestAgentConfig.java @@ -57,7 +57,8 @@ public class TestAgentConfig extends TestCase { checkpointDir.deleteOnExit(); conf.set("chukwaAgent.checkpoint.dir", checkpointDir.getAbsolutePath()); - ChukwaAgent agent = new ChukwaAgent(conf); + ChukwaAgent agent = ChukwaAgent.getAgent(conf); + agent.start(); ConsoleOutConnector conn = new ConsoleOutConnector(agent, true); conn.start(); assertEquals(1, agent.adaptorCount());// check that we processed initial @@ -80,7 +81,8 @@ public class TestAgentConfig extends TestCase { ps.close(); System.out.println("---------------------restarting"); - agent = new ChukwaAgent(conf); + agent = ChukwaAgent.getAgent(conf); + agent.start(); conn = new ConsoleOutConnector(agent, true); conn.start(); assertEquals(2, agent.adaptorCount());// check that we processed initial @@ -113,7 +115,8 @@ public class TestAgentConfig extends TestCase { conf.setInt("chukwaAgent.control.port", 0); System.out.println("\n\n===checkpoints enabled, dir does not exist:"); - ChukwaAgent agent = new ChukwaAgent(conf); + ChukwaAgent agent = ChukwaAgent.getAgent(conf); + agent.start(); assertEquals(0, agent.getAdaptorList().size()); agent.shutdown(); Thread.sleep(2000); @@ -123,7 +126,8 @@ public class TestAgentConfig extends TestCase { System.out .println("\n\n===checkpoints enabled, dir exists but is empty:"); - agent = new ChukwaAgent(conf); + agent = ChukwaAgent.getAgent(conf); + agent.start(); assertEquals(0, agent.getAdaptorList().size()); agent.shutdown(); Thread.sleep(2000); @@ -133,7 +137,8 @@ public class TestAgentConfig extends TestCase { System.out .println("\n\n===checkpoints enabled, dir exists with zero-length file:"); (new File(NONCE_DIR, "chukwa_checkpoint_0")).createNewFile(); - agent = new ChukwaAgent(conf); + agent = ChukwaAgent.getAgent(conf); + agent.start(); assertEquals(0, agent.getAdaptorList().size()); agent.processAddCommand("ADD org.apache.hadoop.chukwa.datacollection.adaptor.ChukwaTestAdaptor testdata 0"); agent.shutdown(); @@ -142,7 +147,8 @@ public class TestAgentConfig extends TestCase { System.out .println("\n\n===checkpoints enabled, dir exists with valid checkpoint"); - agent = new ChukwaAgent(conf); + agent = ChukwaAgent.getAgent(conf); + agent.start(); assertEquals(1, agent.getAdaptorList().size()); agent.shutdown(); Thread.sleep(2000); http://git-wip-us.apache.org/repos/asf/chukwa/blob/6012e14f/src/test/java/org/apache/hadoop/chukwa/datacollection/agent/TestChukwaSsl.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/hadoop/chukwa/datacollection/agent/TestChukwaSsl.java b/src/test/java/org/apache/hadoop/chukwa/datacollection/agent/TestChukwaSsl.java index 898e03c..bddbc8e 100644 --- a/src/test/java/org/apache/hadoop/chukwa/datacollection/agent/TestChukwaSsl.java +++ b/src/test/java/org/apache/hadoop/chukwa/datacollection/agent/TestChukwaSsl.java @@ -86,7 +86,8 @@ public class TestChukwaSsl extends TestCase{ conf.set(SSL_PROTOCOL, sslProtocol); */ //start agent, which starts chukwa rest server - ChukwaAgent agent = new ChukwaAgent(conf); + ChukwaAgent agent = ChukwaAgent.getAgent(conf); + agent.start(); System.out.println("Started ChukwaRestServer"); testSecureRestAdaptor(agent); agent.shutdown(); http://git-wip-us.apache.org/repos/asf/chukwa/blob/6012e14f/src/test/java/org/apache/hadoop/chukwa/datacollection/agent/TestChunkQueue.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/hadoop/chukwa/datacollection/agent/TestChunkQueue.java b/src/test/java/org/apache/hadoop/chukwa/datacollection/agent/TestChunkQueue.java index 66b85e7..051163b 100644 --- a/src/test/java/org/apache/hadoop/chukwa/datacollection/agent/TestChunkQueue.java +++ b/src/test/java/org/apache/hadoop/chukwa/datacollection/agent/TestChunkQueue.java @@ -45,9 +45,9 @@ public class TestChunkQueue extends TestCase { @Override protected void setUp() throws AlreadyRunningException { - agent = ChukwaAgent.getAgent(); if(agent == null){ - agent = new ChukwaAgent(); + agent = ChukwaAgent.getAgent(); + agent.start(); } conf = agent.getConfiguration(); conf.set(CHUNK_QUEUE_LIMIT, Integer.toString(QUEUE_LIMIT)); http://git-wip-us.apache.org/repos/asf/chukwa/blob/6012e14f/src/test/java/org/apache/hadoop/chukwa/datacollection/agent/TestCmd.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/hadoop/chukwa/datacollection/agent/TestCmd.java b/src/test/java/org/apache/hadoop/chukwa/datacollection/agent/TestCmd.java index 6880fce..a34d221 100644 --- a/src/test/java/org/apache/hadoop/chukwa/datacollection/agent/TestCmd.java +++ b/src/test/java/org/apache/hadoop/chukwa/datacollection/agent/TestCmd.java @@ -36,7 +36,8 @@ public class TestCmd extends TestCase { try { Configuration conf = new Configuration(); conf.set("chukwaAgent.control.port", "0"); - agent = new ChukwaAgent(conf); + agent = ChukwaAgent.getAgent(conf); + agent.start(); ConsoleOutConnector conn = new ConsoleOutConnector(agent, true); conn.start(); String l = agent @@ -68,7 +69,8 @@ public class TestCmd extends TestCase { try { Configuration conf = new Configuration(); conf.set("chukwaAgent.control.port", "0"); - agent = new ChukwaAgent(conf); + agent = ChukwaAgent.getAgent(conf); + agent.start(); ConsoleOutConnector conn = new ConsoleOutConnector(agent, true); conn.start(); String name = agent @@ -99,7 +101,8 @@ public class TestCmd extends TestCase { try { Configuration conf = new Configuration(); conf.set("chukwaAgent.control.port", "0"); - agent = new ChukwaAgent(conf); + agent = ChukwaAgent.getAgent(conf); + agent.start(); ConsoleOutConnector conn = new ConsoleOutConnector(agent, true); conn.start(); String n = agent @@ -129,7 +132,8 @@ public class TestCmd extends TestCase { public void testStopAll() throws Exception{ Configuration conf = new Configuration(); conf.set("chukwaAgent.control.port", "0"); - ChukwaAgent agent = new ChukwaAgent(conf); + ChukwaAgent agent = ChukwaAgent.getAgent(conf); + agent.start(); ChunkCatcherConnector chunks = new ChunkCatcherConnector(); chunks.start(); agent.processAddCommand( http://git-wip-us.apache.org/repos/asf/chukwa/blob/6012e14f/src/test/java/org/apache/hadoop/chukwa/datacollection/agent/rest/TestAdaptorController.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/hadoop/chukwa/datacollection/agent/rest/TestAdaptorController.java b/src/test/java/org/apache/hadoop/chukwa/datacollection/agent/rest/TestAdaptorController.java index 0d5bbd0..fa2d5f1 100644 --- a/src/test/java/org/apache/hadoop/chukwa/datacollection/agent/rest/TestAdaptorController.java +++ b/src/test/java/org/apache/hadoop/chukwa/datacollection/agent/rest/TestAdaptorController.java @@ -56,6 +56,7 @@ public class TestAdaptorController extends TestCase { protected void setUp() throws Exception { agent = ChukwaAgent.getAgent(); + agent.start(); ServletHolder servletHolder = new ServletHolder(ServletContainer.class); servletHolder.setInitParameter("com.sun.jersey.config.property.resourceConfigClass", http://git-wip-us.apache.org/repos/asf/chukwa/blob/6012e14f/src/test/java/org/apache/hadoop/chukwa/datacollection/collector/TestAdaptorTimeout.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/hadoop/chukwa/datacollection/collector/TestAdaptorTimeout.java b/src/test/java/org/apache/hadoop/chukwa/datacollection/collector/TestAdaptorTimeout.java index a3ed9f6..2cfd72c 100644 --- a/src/test/java/org/apache/hadoop/chukwa/datacollection/collector/TestAdaptorTimeout.java +++ b/src/test/java/org/apache/hadoop/chukwa/datacollection/collector/TestAdaptorTimeout.java @@ -46,7 +46,8 @@ public class TestAdaptorTimeout extends TestCase { Server collectorServ = TestDelayedAcks.startCollectorOnPort(conf, PORTNO, collector); Thread.sleep(1000); - ChukwaAgent agent = new ChukwaAgent(conf); + ChukwaAgent agent = ChukwaAgent.getAgent(conf); + agent.start(); HttpConnector conn = new HttpConnector(agent, "http://localhost:"+PORTNO+"/";); conn.start(); String resp = agent.processAddCommand("add constSend = " + ConstRateAdaptor.class.getCanonicalName() + http://git-wip-us.apache.org/repos/asf/chukwa/blob/6012e14f/src/test/java/org/apache/hadoop/chukwa/datacollection/collector/TestBackpressure.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/hadoop/chukwa/datacollection/collector/TestBackpressure.java b/src/test/java/org/apache/hadoop/chukwa/datacollection/collector/TestBackpressure.java index b0d4d1c..6423fce 100644 --- a/src/test/java/org/apache/hadoop/chukwa/datacollection/collector/TestBackpressure.java +++ b/src/test/java/org/apache/hadoop/chukwa/datacollection/collector/TestBackpressure.java @@ -56,7 +56,8 @@ public class TestBackpressure extends TestCase { conf.setInt("constAdaptor.minSleep", 50); conf.setInt("chukwaAgent.control.port", 0); - ChukwaAgent agent = new ChukwaAgent(conf); + ChukwaAgent agent = ChukwaAgent.getAgent(conf); + agent.start(); RetryListOfCollectors clist = new RetryListOfCollectors(conf); clist.add("http://localhost:"+PORTNO+"/chukwa";); HttpConnector conn = new HttpConnector(agent); http://git-wip-us.apache.org/repos/asf/chukwa/blob/6012e14f/src/test/java/org/apache/hadoop/chukwa/datacollection/collector/TestDelayedAcks.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/hadoop/chukwa/datacollection/collector/TestDelayedAcks.java b/src/test/java/org/apache/hadoop/chukwa/datacollection/collector/TestDelayedAcks.java index eff7660..b067f16 100644 --- a/src/test/java/org/apache/hadoop/chukwa/datacollection/collector/TestDelayedAcks.java +++ b/src/test/java/org/apache/hadoop/chukwa/datacollection/collector/TestDelayedAcks.java @@ -73,7 +73,8 @@ public class TestDelayedAcks extends TestCase { conf.setInt("chukwaAgent.adaptor.context.switch.time", 500); conf.setInt(AdaptorResetThread.TIMEOUT_OPT, ACK_TIMEOUT); - ChukwaAgent agent = new ChukwaAgent(conf); + ChukwaAgent agent = ChukwaAgent.getAgent(conf); + agent.start(); ChunkCatcherConnector chunks = new ChunkCatcherConnector(); chunks.start(); assertEquals(0, agent.adaptorCount()); @@ -207,7 +208,8 @@ public class TestDelayedAcks extends TestCase { Server collectorServ = startCollectorOnPort(conf, PORTNO, collector); Thread.sleep(1000); - ChukwaAgent agent = new ChukwaAgent(conf); + ChukwaAgent agent = ChukwaAgent.getAgent(conf); + agent.start(); HttpConnector conn = new HttpConnector(agent, "http://localhost:"+PORTNO+"/";); conn.start(); String resp = agent.processAddCommand("add constSend = " + ConstRateAdaptor.class.getCanonicalName() + http://git-wip-us.apache.org/repos/asf/chukwa/blob/6012e14f/src/test/java/org/apache/hadoop/chukwa/datacollection/collector/TestFailedCollectorAck.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/hadoop/chukwa/datacollection/collector/TestFailedCollectorAck.java b/src/test/java/org/apache/hadoop/chukwa/datacollection/collector/TestFailedCollectorAck.java index f5f8d26..de34563 100644 --- a/src/test/java/org/apache/hadoop/chukwa/datacollection/collector/TestFailedCollectorAck.java +++ b/src/test/java/org/apache/hadoop/chukwa/datacollection/collector/TestFailedCollectorAck.java @@ -66,7 +66,8 @@ public class TestFailedCollectorAck extends TestCase { Server collector2_s = TestDelayedAcks.startCollectorOnPort(conf, PORTNO+2, collector2); Thread.sleep(2000); //for collectors to start - ChukwaAgent agent = new ChukwaAgent(conf); + ChukwaAgent agent = ChukwaAgent.getAgent(conf); + agent.start(); HttpConnector conn = new HttpConnector(agent); RetryListOfCollectors clist = new RetryListOfCollectors(conf); clist.add("http://localhost:"+(PORTNO+1)+"/"); http://git-wip-us.apache.org/repos/asf/chukwa/blob/6012e14f/src/test/java/org/apache/hadoop/chukwa/datacollection/controller/TestAgentClient.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/hadoop/chukwa/datacollection/controller/TestAgentClient.java b/src/test/java/org/apache/hadoop/chukwa/datacollection/controller/TestAgentClient.java index 45e29d3..7eaaf05 100644 --- a/src/test/java/org/apache/hadoop/chukwa/datacollection/controller/TestAgentClient.java +++ b/src/test/java/org/apache/hadoop/chukwa/datacollection/controller/TestAgentClient.java @@ -38,7 +38,8 @@ public class TestAgentClient extends TestCase { protected void setUp() throws ChukwaAgent.AlreadyRunningException { config = new Configuration(); - agent = new ChukwaAgent(config); + agent = ChukwaAgent.getAgent(config); + agent.start(); c = new ChukwaAgentController(); connector = new ChunkCatcherConnector(); connector.start(); http://git-wip-us.apache.org/repos/asf/chukwa/blob/6012e14f/src/test/java/org/apache/hadoop/chukwa/validationframework/ChukwaAgentToCollectorValidator.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/hadoop/chukwa/validationframework/ChukwaAgentToCollectorValidator.java b/src/test/java/org/apache/hadoop/chukwa/validationframework/ChukwaAgentToCollectorValidator.java deleted file mode 100644 index a0ca055..0000000 --- a/src/test/java/org/apache/hadoop/chukwa/validationframework/ChukwaAgentToCollectorValidator.java +++ /dev/null @@ -1,269 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.chukwa.validationframework; - - -import java.io.File; -import java.io.IOException; -import org.apache.hadoop.chukwa.conf.ChukwaConfiguration; -import org.apache.hadoop.chukwa.datacollection.agent.ChukwaAgent; -import org.apache.hadoop.chukwa.datacollection.agent.ChukwaAgent.AlreadyRunningException; -import org.apache.hadoop.chukwa.datacollection.collector.CollectorStub; -import org.apache.hadoop.chukwa.datacollection.controller.ChukwaAgentController; -import org.apache.hadoop.chukwa.validationframework.interceptor.ChunkDumper; -import org.apache.hadoop.chukwa.validationframework.interceptor.SetupTestClasses; -import org.apache.hadoop.chukwa.validationframework.util.DataOperations; - -public class ChukwaAgentToCollectorValidator { - public static final int ADD = 100; - public static final int VALIDATE = 200; - - private static void usage() { - System.out.println("usage ..."); - System.exit(-1); - } - - /** - * @param args - * @throws Throwable - * @throws AlreadyRunningException - * @throws IOException - */ - public static void main(String[] args) throws Throwable { - if (args.length != 2) { - usage(); - } - - int command = -1; - - if ("-add".equalsIgnoreCase(args[0])) { - command = ChukwaAgentToCollectorValidator.ADD; - } else if ("-validate".equalsIgnoreCase(args[0])) { - command = ChukwaAgentToCollectorValidator.VALIDATE; - } else { - usage(); - } - - String chukwaTestRepository = System.getenv("chukwaTestRepository"); - if (chukwaTestRepository == null) { - chukwaTestRepository = "/tmp/chukwaTestRepository/"; - } - - if (!chukwaTestRepository.endsWith("/")) { - chukwaTestRepository += "/"; - } - - String fileName = args[1]; - - String name = null; - if (fileName.indexOf("/") >= 0) { - name = fileName.substring(fileName.lastIndexOf("/")); - } else { - name = fileName; - } - - String chukwaTestDirectory = chukwaTestRepository + name; - String inputFile = chukwaTestDirectory + "/input/" + name; - String outputDir = null; - - if (command == ChukwaAgentToCollectorValidator.ADD) { - File dir = new File(chukwaTestDirectory + "/input/"); - if (dir.exists()) { - throw new RuntimeException( - "a test with the same input file is already there, remove it first"); - } - dir.mkdirs(); - DataOperations.copyFile(fileName, inputFile); - outputDir = "/gold"; - } else { - outputDir = "/" + System.currentTimeMillis(); - } - - System.out.println("chukwaTestDirectory [" + chukwaTestDirectory + "]"); - System.out.println("command [" - + ((command == ChukwaAgentToCollectorValidator.ADD) ? "ADD" - : "VALIDATE") + "]"); - System.out.println("fileName [" + inputFile + "]"); - - ChukwaConfiguration conf = new ChukwaConfiguration(true); - String collectorOutputDir = conf.get("chukwaCollector.outputDir"); - - prepareAndSendData(chukwaTestDirectory + outputDir, inputFile, - collectorOutputDir); - extractRawLog(chukwaTestDirectory + outputDir, name, collectorOutputDir); - boolean rawLogTestResult = validateRawLogs(chukwaTestDirectory + outputDir, - name); - - boolean binLogTestResult = true; - - if (command == ChukwaAgentToCollectorValidator.VALIDATE) { - binLogTestResult = validateOutputs(chukwaTestDirectory + outputDir, name); - } - - if (rawLogTestResult == true && binLogTestResult == true) { - System.out.println("test OK"); - System.exit(10); - } else { - System.out.println("test KO"); - throw new RuntimeException("test failed for file [" + name + "]"); - } - } - - public static void prepareAndSendData(String dataRootFolder, - String inputFile, String dataSinkDirectory) throws Throwable { - - ChunkDumper.testRepositoryDumpDir = dataRootFolder + "/"; - - SetupTestClasses.setupClasses(); - - // clean up the collector outputDir. - File collectorDir = new File(dataSinkDirectory); - String[] files = collectorDir.list(); - for (String f : files) { - File file = new File(dataSinkDirectory + File.separator + f); - file.delete(); - System.out.println("Deleting previous collectors files: " + f); - } - - System.out.println("Starting agent"); - String[] agentArgs = new String[0]; - ChukwaAgent.main(agentArgs); - - // Start the collector - System.out.println("Starting collector"); - CollectorStub.main(new String[0]); - - // Start the agent - ChukwaAgent agent = ChukwaAgent.getAgent(); - - int portno = 9093; // Default - ChukwaAgentController cli = new ChukwaAgentController("localhost", portno); - // ADD - // org.apache.hadoop.chukwa.datacollection.adaptor.filetailer. - // CharFileTailingAdaptorUTF8NewLineEscaped - // SysLog - // 0 /var/log/messages - // 0 - System.out.println("Adding adaptor"); - String adaptor = cli.add( - "org.apache.hadoop.chukwa.datacollection.adaptor.filetailer.CharFileTailingAdaptorUTF8NewLineEscaped", - "AutomatedTestType", "0 " + inputFile, 0); - - cli.remove(adaptor); - System.out.println("Adaptor removed"); - agent.shutdown(); - System.out.println("Shutting down agent"); - CollectorStub.jettyServer.stop(); - System.out.println("Shutting down collector"); - Thread.sleep(2000); - } - - public static void extractRawLog(String dataRootFolder, String fileName, - String dataSinkDirectory) throws Exception { - // Adaptor output - DataOperations - .extractRawLogFromDump(dataRootFolder + "/adaptor/", fileName); - // Sender output - DataOperations.extractRawLogFromDump(dataRootFolder + "/sender/", fileName); - - // Collector output - File dir = new File(dataRootFolder + "/collector/"); - dir.mkdirs(); - - File dataSinkDir = new File(dataSinkDirectory); - String[] doneFiles = dataSinkDir.list(); - // Move done file to the final directory - for (String f : doneFiles) { - String outputFile = null; - if (f.endsWith(".done")) { - outputFile = fileName + ".done"; - } else { - outputFile = fileName + ".crc"; - } - System.out.println("Moving that file [" + dataSinkDirectory - + File.separator + f + "] to [" + dataRootFolder + "/collector/" - + outputFile + "]"); - DataOperations.copyFile(dataSinkDirectory + File.separator + f, - dataRootFolder + "/collector/" + outputFile); - } - - DataOperations.extractRawLogFromdataSink(ChunkDumper.testRepositoryDumpDir - + "/collector/", fileName); - } - - public static boolean validateRawLogs(String dataRootFolder, String fileName) { - boolean result = true; - // Validate Adaptor - boolean adaptorMD5 = DataOperations.validateMD5(dataRootFolder - + "/../input/" + fileName, dataRootFolder + "/adaptor/" + fileName - + ".raw"); - if (!adaptorMD5) { - System.out.println("Adaptor validation failed"); - result = false; - } - // Validate Sender - boolean senderMD5 = DataOperations.validateMD5(dataRootFolder - + "/../input/" + fileName, dataRootFolder + "/sender/" + fileName - + ".raw"); - if (!senderMD5) { - System.out.println("Sender validation failed"); - result = false; - } - // Validate DataSink - boolean collectorMD5 = DataOperations.validateMD5(dataRootFolder - + "/../input/" + fileName, dataRootFolder + "/collector/" + fileName - + ".raw"); - if (!collectorMD5) { - System.out.println("collector validation failed"); - result = false; - } - - return result; - } - - public static boolean validateOutputs(String dataRootFolder, String fileName) { - boolean result = true; - // Validate Adaptor - boolean adaptorMD5 = DataOperations.validateMD5(dataRootFolder - + "/../gold/adaptor/" + fileName + ".bin", dataRootFolder + "/adaptor/" - + fileName + ".bin"); - if (!adaptorMD5) { - System.out.println("Adaptor bin validation failed"); - result = false; - } - // Validate Sender - boolean senderMD5 = DataOperations.validateMD5(dataRootFolder - + "/../gold/sender/" + fileName + ".bin", dataRootFolder + "/sender/" - + fileName + ".bin"); - if (!senderMD5) { - System.out.println("Sender bin validation failed"); - result = false; - } - // Validate DataSink - // boolean collectorMD5 = DataOperations.validateRawLog(dataRootFolder + - // "/../gold/collector/" + fileName + ".done", dataRootFolder + - // "/collector/" + fileName + ".done"); - // if (!collectorMD5) - // { - // System.out.println("collector bin validation failed"); - // result = false; - // } - - return result; - } -} \ No newline at end of file
