Maybe because you pass Strings to the LongWritables?
micha
11 Nov. wrote:
> Hi folks,
> I'm writing a little test programm to check the writing speed of DFS
> file system, but can't get the file size using
> "fs.getFileStatus(file).getLen()" or fs.getContentLength(file). Here is my
> code:
>
> package org.apache.hadoop.fs;
> import java.io.IOException;
> import java.io.OutputStream;
>
> import junit.framework.TestCase;
>
> import org.apache.hadoop.conf.Configuration;
> import org.apache.hadoop.fs.FileSystem;
> import org.apache.hadoop.io.LongWritable;
> import org.apache.hadoop.io.SequenceFile;
> import org.apache.hadoop.io.SequenceFile.CompressionType;
> import org.apache.hadoop.io.Text;
>
> public class TestDFSWrite extends TestCase {
> static String ROOT = System.getProperty("test.build.data","fs_test");
> static Path DATA_DIR = new Path(ROOT, "fs_data");
> static long MEGA = 1024 * 1024;
> static int BUFFER_SIZE = 4096;
> static Configuration conf = new Configuration();
> static FileSystem fs;
>
> static byte[] buffer = new byte[BUFFER_SIZE];
> static boolean finished;
>
> public class FileStatusChecker extends Thread{
> Path file;
> Path resultFile;
> int interval;
>
> public FileStatusChecker(Path file, Path resultFile, int interval){
> this.file = file;
> this.resultFile = resultFile;
> this.interval = interval;
> }
>
> public void run(){
> System.out.println("Here is the checker running!!");
> System.out.println(file.toString());
> SequenceFile.Writer writer = null;
> try {
> long lastLen = 1;
> long thisLen = 1;
> writer = SequenceFile.createWriter(fs, conf, resultFile,
> Text.class, Text.class,
> CompressionType.NONE);
>
> while(!finished){
> lastLen = thisLen;
> if(fs.exists(file)){
> System.out.println("File exists!");
> thisLen = fs.getContentLength(file);
> System.out.println("@@"+thisLen);
> }
> else{
> sleep(interval * 10);
> continue;
> }
> long length = thisLen -lastLen;
> System.out.println("thisLen is" + thisLen + "lastLen is " +
> lastLen);
> long cur = System.currentTimeMillis();
> cur = cur - (cur % 10);
> LongWritable current = new LongWritable(cur);
> writer.append(new Text(current.toString()), new Text(new
> LongWritable(length).toString()));
> sleep(interval * 10);
>
> }
> return;
>
> }catch (Exception e) {
> e.printStackTrace();
> }finally {
> try {
> writer.close();
> } catch (Exception e) {
> e.printStackTrace();
> }
> }
> }
> }
>
> public static void main(String[] args) throws Exception {
> {
> try {
> fs = FileSystem.get(conf);
> } catch (IOException e) {
> throw new RuntimeException(e);
> }
> }
> String testFunc = "";
> String fileName = "";
> String resultFileName = "";
> int fileSize = 0;
> int interval = 0;
>
> String usage = "Usage: TestDFSWrite -testfunc [read/write] -file foo
> -size M -interval MS -result resultFile";
> if (args.length == 0) {
> System.err.println(usage);
> System.exit(-1);
> }
>
> for (int i = 0; i < args.length; i++) { // parse command line
> if (args[i].equals("-testfunc")) {
> testFunc = args[++i];
> } else if (args[i].equals("-file")) {
> fileName = args[++i];
> } else if (args[i].equals("-size")) {
> fileSize = Integer.parseInt(args[++i]);
> } else if (args[i].equals("-interval")) {
> interval = Integer.parseInt(args[++i]);
> } else if (args[i].equals("-result")) {
> resultFileName = args[++i];
> }
> }
>
> long total = fileSize * MEGA;
> OutputStream out;
> Path file, resultFile;
> fs.delete(DATA_DIR);
>
> if(testFunc.equals("read")){
> System.out.println("This option is not ready.");
> return;
> }else if(testFunc.equals("write")){
> file = new Path(DATA_DIR, fileName);
> resultFile = new Path(DATA_DIR, resultFileName);
> }else{
> System.out.println("Invalid command line option.");
> return;
> }
>
> FileStatusChecker checker = new TestDFSWrite().new
> FileStatusChecker(file, resultFile, interval);
> System.out.println(file.toString());
> //System.out.println("F L:" +
> newLongWritable(fs.getContentLength(file)).toString());
> out = fs.create(file);
> checker.start();
> long written = 0;
> try {
> finished = false;
> while (written < total) {
> long remains = total - written;
> int length = (remains<=buffer.length) ? (int)remains :
> buffer.length;
> out.write(buffer, 0, length);
> out.flush();
> written += length;
> System.out.println("One segment done!");
> System.out.println("F L:" + new
> LongWritable(fs.getFileStatus(file).getLen()).toString());
> }
> finished =true;
> } finally {
> out.close();
> }
> //System.exit(0);
> }
> }
>
> Neither the inner class FileStatusChecker nor the main function can get the
> right size, they both get "0".
>
> What should be the problem?
>
--
--------------------------------
Michaela Bürgle
Softwareentwicklerin
Forschung und Entwicklung
neofonie GmbH
Robert-Koch-Platz 4
10115 Berlin
fon: +49.30 24627 256
fax: +49.30 24627 120
[EMAIL PROTECTED]
http://www.neofonie.de
Handelsregister
Berlin-Charlottenburg: HRB 67460
Geschaeftsfuehrung
Helmut Hoffer von Ankershoffen
Nurhan Yildirim
--------------------------------
