Author: johan
Date: Thu Oct 23 02:36:44 2008
New Revision: 707328
URL: http://svn.apache.org/viewvc?rev=707328&view=rev
Log:
HADOOP-4431. Add versionning/tags to Chukwa Chunk.
(Jerome Boulon via Johan)
Added:
hadoop/core/trunk/src/contrib/chukwa/src/test/org/apache/hadoop/chukwa/ChunkImplTest.java
Modified:
hadoop/core/trunk/CHANGES.txt
hadoop/core/trunk/src/contrib/chukwa/build.xml
hadoop/core/trunk/src/contrib/chukwa/conf/chukwa-agent-conf.xml
hadoop/core/trunk/src/contrib/chukwa/conf/chukwa-agent-conf.xml.template
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/Chunk.java
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/ChunkImpl.java
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/datacollection/agent/ChukwaAgent.java
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/extraction/engine/datasource/database/DatabaseDS.java
hadoop/core/trunk/src/contrib/chukwa/src/test/org/apache/hadoop/chukwa/TestChunkBuilder.java
hadoop/core/trunk/src/contrib/chukwa/src/test/org/apache/hadoop/chukwa/inputtools/log4j/TestChukwaAppender.java
Modified: hadoop/core/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/core/trunk/CHANGES.txt?rev=707328&r1=707327&r2=707328&view=diff
==============================================================================
--- hadoop/core/trunk/CHANGES.txt (original)
+++ hadoop/core/trunk/CHANGES.txt Thu Oct 23 02:36:44 2008
@@ -45,6 +45,9 @@
HADOOP-4284. Support filters that apply to all requests, or global filters,
to HttpServer. (Kan Zhang via cdouglas)
+
+ HADOOP-4431. Add versionning/tags to Chukwa Chunk.
+ (Jerome Boulon via Johan)
OPTIMIZATIONS
Modified: hadoop/core/trunk/src/contrib/chukwa/build.xml
URL:
http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/build.xml?rev=707328&r1=707327&r2=707328&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/chukwa/build.xml (original)
+++ hadoop/core/trunk/src/contrib/chukwa/build.xml Thu Oct 23 02:36:44 2008
@@ -19,22 +19,48 @@
<project name="chukwa" default="main">
+ <import file="../build-contrib.xml"/>
+
+
<property name="chukwaVersion" value="0.0.1"/>
<property name="lib.dir" value="${basedir}/lib"/>
<property name="src.dir" value="${basedir}/src"/>
+ <property name="build.classes" value="${build.dir}/classes"/>
<property name="conf.dir" value="${basedir}/conf"/>
<property name="docs.dir" value="${basedir}/docs"/>
<property name="tools.dir" value="${basedir}/tools"/>
<property name="dist.dir" value="${basedir}/dist"/>
<property name="opt.dir" value="${basedir}/opt"/>
<property name="javac.debug" value="on"/>
-
- <import file="../build-contrib.xml"/>
+ <property name="javac.version" value="1.6"/>
+
+ <property name="test.src.dir" value="${basedir}/src/test"/>
+ <property name="test.lib.dir" value="${basedir}/src/test/lib"/>
+ <property name="test.build.dir" value="${build.dir}/test"/>
+ <property name="test.generated.dir" value="${test.build.dir}/src"/>
+ <property name="test.build.data" value="${test.build.dir}/data"/>
+ <property name="test.cache.data" value="${test.build.dir}/cache"/>
+ <property name="test.debug.data" value="${test.build.dir}/debug"/>
+ <property name="test.log.dir" value="${test.build.dir}/logs"/>
+ <property name="test.build.classes" value="${test.build.dir}/classes"/>
+ <property name="test.build.testjar" value="${test.build.dir}/testjar"/>
+ <property name="test.include" value="Test*"/>
+ <property name="test.classpath.id" value="test.classpath"/>
+ <property name="test.output" value="yes"/>
+ <property name="test.timeout" value="900000"/>
+ <property name="test.junit.output.format" value="plain"/>
+ <property name="test.junit.fork.mode" value="perTest" />
+ <property name="test.junit.printsummary" value="yes" />
+ <property name="test.junit.haltonfailure" value="yes" />
+ <property name="test.junit.maxmemory" value="256m" />
+
<target name="init">
<mkdir dir="${build.dir}"/>
+ <mkdir dir="${build.classes}"/>
<mkdir dir="${build.dir}/test"/>
+
<!--
<copy todir="${build.dir}">
<fileset dir="${basedir}">
@@ -59,16 +85,49 @@
</target>
<target name="compile" depends="init" description="Compilation target">
- <javac srcdir="src/java/org/apache/hadoop/chukwa"
destdir="${build.dir}" excludes="**/ChukwaTTInstru.java" debug="${javac.debug}">
+ <javac srcdir="src/java/org/apache/hadoop/chukwa"
destdir="${build.classes}" excludes="**/ChukwaTTInstru.java"
debug="${javac.debug}">
<classpath refid="chukwaHadoopClasspath" />
</javac>
</target>
- <target name="test" depends="init" description="Test target">
- <javac srcdir="src/test/org/apache/hadoop/chukwa"
destdir="${build.dir}" excludes="**/TestDemux.java" debug="${javac.debug}">
- <classpath refid="chukwaHadoopClasspath" />
- </javac>
- </target>
+ <target name="compile-test" depends="init" description="Test target">
+
+ <delete dir="${test.build.dir}"/>
+ <mkdir dir="${test.build.dir}"/>
+ <delete dir="${test.log.dir}"/>
+ <mkdir dir="${test.log.dir}"/>
+ <delete dir="${test.build.classes}"/>
+ <mkdir dir="${test.build.classes}"/>
+
+ <javac srcdir="${test.src.dir}/org/apache/hadoop/chukwa"
destdir="${test.build.dir}/classes" excludes="**/TestDemux.java"
debug="${javac.debug}">
+ <classpath refid="chukwaTestClasspath" />
+ </javac>
+ </target>
+
+
+
+ <target name="test" depends="compile,compile-test,test-chunk"
description="Chukwa Tests">
+ </target>
+
+ <target name="test-chunk" depends="compile,compile-test"
description="Test chunk">
+
+ <junit showoutput="yes"
+ fork="yes"
+ printsummary="${test.junit.printsummary}"
+ forkmode="${test.junit.fork.mode}"
+ maxmemory="${test.junit.maxmemory}"
+ dir="${test.build.dir}/classes/" timeout="${test.timeout}"
+ errorProperty="tests.failed" failureProperty="tests.failed">
+ <classpath refid="chukwaTestClasspath"/>
+ <sysproperty key="test.src.dir" value="${test.src.dir}"/>
+ <formatter type="${test.junit.output.format}" />
+ <batchtest todir="${test.build.dir}" >
+ <fileset dir="${test.src.dir}"
includes="org/apache/hadoop/chukwa/*.java"/>
+ </batchtest>
+ </junit>
+ <fail if="tests.failed">Tests failed!</fail>
+
+ </target>
<target name="collector" depends="compile" description="Prepare
collector.war">
<mkdir dir="${build.dir}/collector"/>
@@ -359,5 +418,18 @@
<exclude name="**/excluded/" />
</fileset>
</path>
+ <path id="chukwaTestClasspath">
+ <pathelement location="${hadoop.root}/build/classes"/>
+ <pathelement
location="${hadoop.root}/build/contrib/chukwa/classes/"/>
+ <pathelement
location="${hadoop.root}/build/contrib/chukwa/test/classes/"/>
+ <fileset dir="${lib.dir}">
+ <include name="**/*.jar" />
+ <exclude name="**/excluded/" />
+ </fileset>
+ <fileset dir="${hadoop.root}/lib">
+ <include name="**/*.jar" />
+ <exclude name="**/excluded/" />
+ </fileset>
+ </path>
</project>
Modified: hadoop/core/trunk/src/contrib/chukwa/conf/chukwa-agent-conf.xml
URL:
http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/conf/chukwa-agent-conf.xml?rev=707328&r1=707327&r2=707328&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/chukwa/conf/chukwa-agent-conf.xml (original)
+++ hadoop/core/trunk/src/contrib/chukwa/conf/chukwa-agent-conf.xml Thu Oct 23
02:36:44 2008
@@ -17,4 +17,11 @@
<description>The hostname of the agent on this node. Usually localhost,
this is used by the chukwa instrumentation agent-control interface
library</description>
</property>
+ <property>
+ <name>chukwaAgent.tags</name>
+ <value>cluster="demo"</value>
+ <description>The cluster's name for this agent</description>
+ </property>
+
+
</configuration>
Modified:
hadoop/core/trunk/src/contrib/chukwa/conf/chukwa-agent-conf.xml.template
URL:
http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/conf/chukwa-agent-conf.xml.template?rev=707328&r1=707327&r2=707328&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/chukwa/conf/chukwa-agent-conf.xml.template
(original)
+++ hadoop/core/trunk/src/contrib/chukwa/conf/chukwa-agent-conf.xml.template
Thu Oct 23 02:36:44 2008
@@ -18,6 +18,12 @@
</property>
<property>
+ <name>chukwaAgent.tags</name>
+ <value>cluster="demo"</value>
+ <description>The cluster's name for this agent</description>
+ </property>
+
+ <property>
<name>chukwaAgent.checkpoint.name</name>
<value>chukwa_agent_checkpoint</value>
<description>the prefix to to prepend to the agent's checkpoint
file(s)</description>
Modified:
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/Chunk.java
URL:
http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/Chunk.java?rev=707328&r1=707327&r2=707328&view=diff
==============================================================================
---
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/Chunk.java
(original)
+++
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/Chunk.java
Thu Oct 23 02:36:44 2008
@@ -92,5 +92,17 @@
*/
public int getSerializedSizeEstimate();
+/**
+ * @return tags from this chunk.
+ *
+ */
+ public String getTags();
+
+ /**
+ * Set tags to this chunk.
+ *
+ */
+ public void setTags(String tags);
+
public void write(DataOutput data) throws IOException;
}
Modified:
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/ChunkImpl.java
URL:
http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/ChunkImpl.java?rev=707328&r1=707327&r2=707328&view=diff
==============================================================================
---
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/ChunkImpl.java
(original)
+++
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/ChunkImpl.java
Thu Oct 23 02:36:44 2008
@@ -23,25 +23,28 @@
import java.io.IOException;
import java.net.InetAddress;
import java.net.UnknownHostException;
-import java.util.ArrayList;
import org.apache.hadoop.chukwa.datacollection.adaptor.Adaptor;
+import org.apache.hadoop.chukwa.datacollection.agent.ChukwaAgent;
public class ChunkImpl implements org.apache.hadoop.io.Writable, Chunk
{
-
+ public static int PROTOCOL_VERSION=1;
+
private String source = "";
private String application = "";
private String dataType = "";
+ private String tags = "";
private byte[] data = null;
private int[] recordEndOffsets;
+ private int protocolVersion=1;
+ private long seqID;
private String debuggingInfo="";
-
private transient Adaptor initiator;
- long seqID;
ChunkImpl() {
+ this.tags = ChukwaAgent.getTags();
}
public static ChunkImpl getBlankChunk() {
@@ -50,11 +53,12 @@
public ChunkImpl(String dataType, String streamName, long seq, byte[] data,
Adaptor source) {
this.seqID = seq;
+ this.source = localHostAddr;
+ this.tags = ChukwaAgent.getTags();
this.application = streamName;
this.dataType = dataType;
this.data = data;
this.initiator = source;
- this.source = localHostAddr;
}
/**
@@ -109,6 +113,13 @@
seqID=l;
}
+ public int getProtocolVersion() {
+ return protocolVersion;
+ }
+
+ public void setProtocolVersion(int pv) {
+ this.protocolVersion = pv;
+ }
public String getApplication(){
return application;
}
@@ -149,12 +160,30 @@
dataType = t;
}
+ @Override
+ public void setTags(String tags)
+ {
+ this.tags = tags;
+ }
+
+/**
+ * @see org.apache.hadoop.chukwa.Chunk#getTags()
+ */
+ public String getTags() {
+ return tags;
+ }
+
/**
* @see org.apache.hadoop.io.Writable#readFields(java.io.DataInput)
*/
public void readFields(DataInput in) throws IOException {
+ setProtocolVersion(in.readInt());
+ if(protocolVersion!=PROTOCOL_VERSION) {
+ throw new IOException("Protocol version mismatched, drop data.
source version: "+protocolVersion+", collector version:"+PROTOCOL_VERSION);
+ }
setSeqID(in.readLong());
setSource(in.readUTF());
+ tags = in.readUTF(); //no public set method here
setApplication(in.readUTF());
setDataType(in.readUTF());
setDebugInfo(in.readUTF());
@@ -167,12 +196,15 @@
in.readFully(data);
}
+
/**
* @see org.apache.hadoop.io.Writable#write(java.io.DataOutput)
*/
public void write(DataOutput out) throws IOException {
+ out.writeInt(PROTOCOL_VERSION);
out.writeLong(seqID);
out.writeUTF(source);
+ out.writeUTF(tags);
out.writeUTF(application);
out.writeUTF(dataType);
out.writeUTF(debuggingInfo);
Modified:
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/datacollection/agent/ChukwaAgent.java
URL:
http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/datacollection/agent/ChukwaAgent.java?rev=707328&r1=707327&r2=707328&view=diff
==============================================================================
---
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/datacollection/agent/ChukwaAgent.java
(original)
+++
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/datacollection/agent/ChukwaAgent.java
Thu Oct 23 02:36:44 2008
@@ -40,7 +40,8 @@
{
boolean DO_CHECKPOINT_RESTORE = false;
boolean WRITE_CHECKPOINTS = false;
-
+
+ static String tags = "";
static Logger log = Logger.getLogger(ChukwaAgent.class);
//doesn't need an equals(), comparator, etc
@@ -444,6 +445,8 @@
}
// String initialAdaptorsStr = conf.get("initial_adaptors_file");
+ tags = conf.get("chukwaAgent.tags", "cluster=\"unknown\"");
+
initialAdaptors = new File(chukwaHome + "conf/initial_adaptors");
}
@@ -495,4 +498,8 @@
return controlSock;
}
+ public static String getTags() {
+ return tags;
+ }
+
}
Modified:
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/extraction/engine/datasource/database/DatabaseDS.java
URL:
http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/extraction/engine/datasource/database/DatabaseDS.java?rev=707328&r1=707327&r2=707328&view=diff
==============================================================================
---
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/extraction/engine/datasource/database/DatabaseDS.java
(original)
+++
hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/extraction/engine/datasource/database/DatabaseDS.java
Thu Oct 23 02:36:44 2008
@@ -51,7 +51,7 @@
if(cluster==null)
{
- cluster="mithrilgold";
+ cluster="demo";
}
if (dataSource.equalsIgnoreCase("MRJob"))
Added:
hadoop/core/trunk/src/contrib/chukwa/src/test/org/apache/hadoop/chukwa/ChunkImplTest.java
URL:
http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/src/test/org/apache/hadoop/chukwa/ChunkImplTest.java?rev=707328&view=auto
==============================================================================
---
hadoop/core/trunk/src/contrib/chukwa/src/test/org/apache/hadoop/chukwa/ChunkImplTest.java
(added)
+++
hadoop/core/trunk/src/contrib/chukwa/src/test/org/apache/hadoop/chukwa/ChunkImplTest.java
Thu Oct 23 02:36:44 2008
@@ -0,0 +1,58 @@
+package org.apache.hadoop.chukwa;
+
+import java.io.IOException;
+
+import junit.framework.TestCase;
+
+import org.apache.hadoop.io.DataInputBuffer;
+import org.apache.hadoop.io.DataOutputBuffer;
+
+public class ChunkImplTest extends TestCase
+{
+ public void testVersion()
+ {
+ ChunkBuilder cb = new ChunkBuilder();
+ cb.addRecord("foo".getBytes());
+ cb.addRecord("bar".getBytes());
+ cb.addRecord("baz".getBytes());
+ Chunk c = cb.getChunk();
+ DataOutputBuffer ob = new
DataOutputBuffer(c.getSerializedSizeEstimate());
+ try
+ {
+ c.write(ob);
+ DataInputBuffer ib = new DataInputBuffer();
+ ib.reset(ob.getData(), c.getSerializedSizeEstimate());
+ int version = ib.readInt();
+ assertEquals(version,ChunkImpl.PROTOCOL_VERSION);
+ }
+ catch (IOException e)
+ {
+ e.printStackTrace();
+ fail("Should nor raise any exception");
+ }
+ }
+
+ public void testWrongVersion()
+ {
+ ChunkBuilder cb = new ChunkBuilder();
+ cb.addRecord("foo".getBytes());
+ cb.addRecord("bar".getBytes());
+ cb.addRecord("baz".getBytes());
+ Chunk c = cb.getChunk();
+ DataOutputBuffer ob = new
DataOutputBuffer(c.getSerializedSizeEstimate());
+ try
+ {
+ c.write(ob);
+ DataInputBuffer ib = new DataInputBuffer();
+ ib.reset(ob.getData(), c.getSerializedSizeEstimate());
+ //change current chunkImpl version
+ ChunkImpl.PROTOCOL_VERSION =
ChunkImpl.PROTOCOL_VERSION+1;
+ ChunkImpl.read(ib);
+ fail("Should have raised an IOexception");
+ }
+ catch (IOException e)
+ {
+ // right behavior, do nothing
+ }
+ }
+}
Modified:
hadoop/core/trunk/src/contrib/chukwa/src/test/org/apache/hadoop/chukwa/TestChunkBuilder.java
URL:
http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/src/test/org/apache/hadoop/chukwa/TestChunkBuilder.java?rev=707328&r1=707327&r2=707328&view=diff
==============================================================================
---
hadoop/core/trunk/src/contrib/chukwa/src/test/org/apache/hadoop/chukwa/TestChunkBuilder.java
(original)
+++
hadoop/core/trunk/src/contrib/chukwa/src/test/org/apache/hadoop/chukwa/TestChunkBuilder.java
Thu Oct 23 02:36:44 2008
@@ -35,5 +35,4 @@
assertEquals(5, chunk.getRecordOffsets()[1]);
assertEquals(8, chunk.getRecordOffsets()[2]);
}
-
}
Modified:
hadoop/core/trunk/src/contrib/chukwa/src/test/org/apache/hadoop/chukwa/inputtools/log4j/TestChukwaAppender.java
URL:
http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/src/test/org/apache/hadoop/chukwa/inputtools/log4j/TestChukwaAppender.java?rev=707328&r1=707327&r2=707328&view=diff
==============================================================================
---
hadoop/core/trunk/src/contrib/chukwa/src/test/org/apache/hadoop/chukwa/inputtools/log4j/TestChukwaAppender.java
(original)
+++
hadoop/core/trunk/src/contrib/chukwa/src/test/org/apache/hadoop/chukwa/inputtools/log4j/TestChukwaAppender.java
Thu Oct 23 02:36:44 2008
@@ -29,7 +29,8 @@
public class TestChukwaAppender extends TestCase {
- public void testChukwaAppender() {
+ @SuppressWarnings("deprecation")
+public void testChukwaAppender() {
try {
ChukwaAgent agent = new ChukwaAgent();
