No problem. I didn't test on jdk8 machine yet too. On Thu, Apr 9, 2015 at 9:51 PM, Tommaso Teofili <[email protected]> wrote: > thanks for having take care of that yourself Edward, I just didn't have > time to do it myself yet. > > Regards, > Tommaso > > 2015-04-09 14:15 GMT+02:00 <[email protected]>: > >> Author: edwardyoon >> Date: Thu Apr 9 12:15:13 2015 >> New Revision: 1672307 >> >> URL: http://svn.apache.org/r1672307 >> Log: >> Fix YarnSerializePrinting and pom file for jdk8 >> >> Modified: >> hama/trunk/pom.xml >> hama/trunk/yarn/src/main/java/org/apache/hama/bsp/YARNBSPJobClient.java >> >> hama/trunk/yarn/src/main/java/org/apache/hama/bsp/YarnSerializePrinting.java >> >> Modified: hama/trunk/pom.xml >> URL: >> http://svn.apache.org/viewvc/hama/trunk/pom.xml?rev=1672307&r1=1672306&r2=1672307&view=diff >> >> ============================================================================== >> --- hama/trunk/pom.xml (original) >> +++ hama/trunk/pom.xml Thu Apr 9 12:15:13 2015 >> @@ -126,6 +126,24 @@ >> >> <profiles> >> <profile> >> + <id>doclint-java8-disable</id> >> + <activation> >> + <jdk>[1.8,)</jdk> >> + </activation> >> + <build> >> + <plugins> >> + <plugin> >> + <groupId>org.apache.maven.plugins</groupId> >> + <artifactId>maven-javadoc-plugin</artifactId> >> + <configuration> >> + <additionalparam>-Xdoclint:none</additionalparam> >> + </configuration> >> + </plugin> >> + </plugins> >> + </build> >> + </profile> >> + >> + <profile> >> <id>hadoop1</id> >> >> <modules> >> @@ -476,7 +494,6 @@ >> <configuration> >> <aggregate>true</aggregate> >> <outputDirectory>docs/apidocs</outputDirectory> >> - <additionalparam>-Xdoclint:none</additionalparam> >> </configuration> >> <executions> >> <execution> >> >> Modified: >> hama/trunk/yarn/src/main/java/org/apache/hama/bsp/YARNBSPJobClient.java >> URL: >> http://svn.apache.org/viewvc/hama/trunk/yarn/src/main/java/org/apache/hama/bsp/YARNBSPJobClient.java?rev=1672307&r1=1672306&r2=1672307&view=diff >> >> ============================================================================== >> --- >> hama/trunk/yarn/src/main/java/org/apache/hama/bsp/YARNBSPJobClient.java >> (original) >> +++ >> hama/trunk/yarn/src/main/java/org/apache/hama/bsp/YARNBSPJobClient.java Thu >> Apr 9 12:15:13 2015 >> @@ -424,7 +424,6 @@ public class YARNBSPJobClient extends BS >> return files; >> } >> >> - >> private void addToLocalResources(FileSystem fs, String fileSrcPath, >> String fileDstPath, String fileName, Map<String, LocalResource> >> localResources) >> throws IOException { >> >> Modified: >> hama/trunk/yarn/src/main/java/org/apache/hama/bsp/YarnSerializePrinting.java >> URL: >> http://svn.apache.org/viewvc/hama/trunk/yarn/src/main/java/org/apache/hama/bsp/YarnSerializePrinting.java?rev=1672307&r1=1672306&r2=1672307&view=diff >> >> ============================================================================== >> --- >> hama/trunk/yarn/src/main/java/org/apache/hama/bsp/YarnSerializePrinting.java >> (original) >> +++ >> hama/trunk/yarn/src/main/java/org/apache/hama/bsp/YarnSerializePrinting.java >> Thu Apr 9 12:15:13 2015 >> @@ -21,39 +21,62 @@ import java.io.IOException; >> >> import org.apache.commons.logging.Log; >> import org.apache.commons.logging.LogFactory; >> +import org.apache.hadoop.fs.FSDataInputStream; >> +import org.apache.hadoop.fs.FileStatus; >> +import org.apache.hadoop.fs.FileSystem; >> +import org.apache.hadoop.fs.Path; >> +import org.apache.hadoop.io.IntWritable; >> +import org.apache.hadoop.io.IOUtils; >> import org.apache.hadoop.io.NullWritable; >> -import org.apache.hama.Constants; >> +import org.apache.hadoop.io.Text; >> import org.apache.hama.HamaConfiguration; >> import org.apache.hama.bsp.sync.SyncException; >> >> public class YarnSerializePrinting { >> >> + public static Path OUTPUT_PATH = new Path("/tmp/serialout"); >> + >> public static class HelloBSP extends >> - BSP<NullWritable, NullWritable, NullWritable, NullWritable, >> NullWritable> { >> + BSP<NullWritable, NullWritable, IntWritable, Text, NullWritable> { >> public static final Log LOG = LogFactory.getLog(HelloBSP.class); >> - private final static int PRINT_INTERVAL = 1000; >> private int num; >> >> @Override >> public void bsp( >> - BSPPeer<NullWritable, NullWritable, NullWritable, NullWritable, >> NullWritable> bspPeer) >> + BSPPeer<NullWritable, NullWritable, IntWritable, Text, >> NullWritable> bspPeer) >> throws IOException, SyncException, InterruptedException { >> num = bspPeer.getConfiguration().getInt("bsp.peers.num", 1); >> - LOG.info(bspPeer.getAllPeerNames()); >> + IntWritable peerNum = new IntWritable(); >> + Text txt = new Text(); >> int i = 0; >> for (String otherPeer : bspPeer.getAllPeerNames()) { >> String peerName = bspPeer.getPeerName(); >> if (peerName.equals(otherPeer)) { >> - LOG.info("Hello BSP from " + (i + 1) + " of " + num + ": " + >> peerName); >> + peerNum.set(i); >> + txt.set("Hello BSP from " + (i + 1) + " of " + num + ": " + >> peerName); >> + bspPeer.write(null, txt); >> } >> >> - Thread.sleep(PRINT_INTERVAL); >> bspPeer.sync(); >> i++; >> } >> } >> } >> >> + static void printOutput(HamaConfiguration conf) throws IOException { >> + FileSystem fs = FileSystem.get(conf); >> + FileStatus[] files = fs.listStatus(OUTPUT_PATH); >> + for (FileStatus file : files) { >> + if (file.getLen() > 0) { >> + FSDataInputStream in = fs.open(file.getPath()); >> + IOUtils.copyBytes(in, System.out, conf, false); >> + in.close(); >> + } >> + } >> + >> + //fs.delete(OUTPUT_PATH, true); >> + } >> + >> public static void main(String[] args) throws IOException, >> InterruptedException, ClassNotFoundException { >> HamaConfiguration conf = new HamaConfiguration(); >> @@ -63,9 +86,17 @@ public class YarnSerializePrinting { >> job.setJarByClass(HelloBSP.class); >> job.setJobName("Serialize Printing"); >> job.setInputFormat(NullInputFormat.class); >> - job.setOutputFormat(NullOutputFormat.class); >> + job.setOutputFormat(TextOutputFormat.class); >> + job.setOutputKeyClass(IntWritable.class); >> + job.setOutputValueClass(Text.class); >> + job.setOutputPath(OUTPUT_PATH); >> job.setMemoryUsedPerTaskInMb(100); >> job.setNumBspTask(4); >> + >> + long startTime = System.currentTimeMillis(); >> job.waitForCompletion(true); >> + printOutput(conf); >> + System.out.println("Job Finished in " >> + + (System.currentTimeMillis() - startTime) / 1000.0 + " >> seconds"); >> } >> } >> >> >>
-- Best Regards, Edward J. Yoon
