Added: hadoop/core/trunk/src/test/org/apache/hadoop/fs/loadGenerator/LoadGenerator.java URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/test/org/apache/hadoop/fs/loadGenerator/LoadGenerator.java?rev=694458&view=auto ============================================================================== --- hadoop/core/trunk/src/test/org/apache/hadoop/fs/loadGenerator/LoadGenerator.java (added) +++ hadoop/core/trunk/src/test/org/apache/hadoop/fs/loadGenerator/LoadGenerator.java Thu Sep 11 13:23:09 2008 @@ -0,0 +1,466 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.loadGenerator; + +import java.io.IOException; +import java.io.InputStream; +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.util.ArrayList; +import java.util.Random; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.conf.Configured; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.util.Tool; +import org.apache.hadoop.util.ToolRunner; + +/** The load generator is a tool for testing NameNode behavior under + * different client loads. + * It allows the user to generate different mixes of read, write, + * and list requests by specifying the probabilities of read and + * write. The user controls the the intensity of the load by + * adjusting parameters for the number of worker threads and the delay + * between operations. While load generators are running, the user + * can profile and monitor the running of the NameNode. When a load + * generator exits, it print some NameNode statistics like the average + * execution time of each kind of operations and the NameNode + * throughput. + * + * After command line argument parsing and data initialization, + * the load generator spawns the number of worker threads + * as specified by the user. + * Each thread sends a stream of requests to the NameNode. + * For each iteration, it first decides if it is going to read a file, + * create a file, or listing a directory following the read and write + * probabilities specified by the user. + * When reading, it randomly picks a file in the test space and reads + * the entire file. When writing, it randomly picks a directory in the + * test space and creates a file whose name consists of the current + * machine's host name and the thread id. The length of the file + * follows Gaussian distribution with an average size of 2 blocks and + * the standard deviation of 1 block. The new file is filled with 'a'. + * Immediately after the file creation completes, the file is deleted + * from the test space. + * While listing, it randomly picks a directory in the test space and + * list the directory content. + * Between two consecutive operations, the thread pauses for a random + * amount of time in the range of [0, maxDelayBetweenOps] + * if the specified max delay is not zero. + * All threads are stopped when the specified elapsed time is passed. + * Before exiting, the program prints the average execution for + * each kind of NameNode operations, and the number of requests + * served by the NameNode. + * + * The synopsis of the command is + * java LoadGenerator + * -readProbability <read probability>: read probability [0, 1] + * with a default value of 0.3333. + * -writeProbability <write probability>: write probability [0, 1] + * with a default value of 0.3333. + * -root <root>: test space with a default value of /testLoadSpace + * -maxDelayBetweenOps <maxDelayBetweenOpsInMillis>: + * Max delay in the unit of milliseconds between two operations with a + * default value of 0 indicating no delay. + * -numOfThreads <numOfThreads>: + * number of threads to spawn with a default value of 200. + * -elapsedTime <elapsedTimeInSecs>: + * the elapsed time of program with a default value of 0 + * indicating running forever + * -startTime <startTimeInMillis> : when the threads start to run. + */ +public class LoadGenerator extends Configured implements Tool { + private volatile boolean shouldRun = true; + private Path root = DataGenerator.DEFAULT_ROOT; + private FileSystem fs; + private int maxDelayBetweenOps = 0; + private int numOfThreads = 200; + private double readPr = 0.3333; + private double writePr = 0.3333; + private long elapsedTime = 0; + private long startTime = System.currentTimeMillis()+10000; + final static private int BLOCK_SIZE = 10; + private ArrayList<String> files = new ArrayList<String>(); // a table of file names + private ArrayList<String> dirs = new ArrayList<String>(); // a table of directory names + private Random r = null; + final private static String USAGE = "java LoadGenerator\n" + + "-readProbability <read probability>\n" + + "-writeProbability <write probability>\n" + + "-root <root>\n" + + "-maxDelayBetweenOps <maxDelayBetweenOpsInMillis>\n" + + "-numOfThreads <numOfThreads>\n" + + "-elapsedTime <elapsedTimeInSecs>\n" + + "-startTime <startTimeInMillis>"; + final private String hostname; + + /** Constructor */ + public LoadGenerator() throws IOException, UnknownHostException { + InetAddress addr = InetAddress.getLocalHost(); + hostname = addr.getHostName(); + } + + private final static int OPEN = 0; + private final static int LIST = 1; + private final static int CREATE = 2; + private final static int WRITE_CLOSE = 3; + private final static int DELETE = 4; + private final static int TOTAL_OP_TYPES =5; + private long [] executionTime = new long[TOTAL_OP_TYPES]; + private long [] totalNumOfOps = new long[TOTAL_OP_TYPES]; + + /** A thread sends a stream of requests to the NameNode. + * At each iteration, it first decides if it is going to read a file, + * create a file, or listing a directory following the read + * and write probabilities. + * When reading, it randomly picks a file in the test space and reads + * the entire file. When writing, it randomly picks a directory in the + * test space and creates a file whose name consists of the current + * machine's host name and the thread id. The length of the file + * follows Gaussian distribution with an average size of 2 blocks and + * the standard deviation of 1 block. The new file is filled with 'a'. + * Immediately after the file creation completes, the file is deleted + * from the test space. + * While listing, it randomly picks a directory in the test space and + * list the directory content. + * Between two consecutive operations, the thread pauses for a random + * amount of time in the range of [0, maxDelayBetweenOps] + * if the specified max delay is not zero. + * A thread runs for the specified elapsed time if the time isn't zero. + * Otherwise, it runs forever. + */ + private class DFSClientThread extends Thread { + private int id; + private long [] executionTime = new long[TOTAL_OP_TYPES]; + private long [] totalNumOfOps = new long[TOTAL_OP_TYPES]; + private byte[] buffer = new byte[1024]; + + private DFSClientThread(int id) { + this.id = id; + } + + /** Main loop + * Each iteration decides what's the next operation and then pauses. + */ + public void run() { + try { + while (shouldRun) { + nextOp(); + delay(); + } + } catch (Exception ioe) { + System.err.println(ioe.getLocalizedMessage()); + ioe.printStackTrace(); + } + } + + /** Let the thread pause for a random amount of time in the range of + * [0, maxDelayBetweenOps] if the delay is not zero. Otherwise, no pause. + */ + private void delay() throws InterruptedException { + if (maxDelayBetweenOps>0) { + int delay = r.nextInt(maxDelayBetweenOps); + Thread.sleep(delay); + } + } + + /** Perform the next operation. + * + * Depending on the read and write probabilities, the next + * operation could be either read, write, or list. + */ + private void nextOp() throws IOException { + double rn = r.nextDouble(); + if (rn < readPr) { + read(); + } else if (rn < readPr+writePr) { + write(); + } else { + list(); + } + } + + /** Read operation randomly picks a file in the test space and reads + * the entire file */ + private void read() throws IOException { + String fileName = files.get(r.nextInt(files.size())); + long startTime = System.currentTimeMillis(); + InputStream in = fs.open(new Path(fileName)); + executionTime[OPEN] += (System.currentTimeMillis()-startTime); + totalNumOfOps[OPEN]++; + while (in.read(buffer) != -1) {} + in.close(); + } + + /** The write operation randomly picks a directory in the + * test space and creates a file whose name consists of the current + * machine's host name and the thread id. The length of the file + * follows Gaussian distribution with an average size of 2 blocks and + * the standard deviation of 1 block. The new file is filled with 'a'. + * Immediately after the file creation completes, the file is deleted + * from the test space. + */ + private void write() throws IOException { + String dirName = dirs.get(r.nextInt(dirs.size())); + Path file = new Path(dirName, hostname+id); + double fileSize = 0; + while ((fileSize = r.nextGaussian()+2)<=0) {} + genFile(file, (long)(fileSize*BLOCK_SIZE)); + long startTime = System.currentTimeMillis(); + fs.delete(file, true); + executionTime[DELETE] += (System.currentTimeMillis()-startTime); + totalNumOfOps[DELETE]++; + } + + /** The list operation randomly picks a directory in the test space and + * list the directory content. + */ + private void list() throws IOException { + String dirName = dirs.get(r.nextInt(dirs.size())); + long startTime = System.currentTimeMillis(); + fs.listStatus(new Path(dirName)); + executionTime[LIST] += (System.currentTimeMillis()-startTime); + totalNumOfOps[LIST]++; + } + } + + /** Main function: + * It first initializes data by parsing the command line arguments. + * It then starts the number of DFSClient threads as specified by + * the user. + * It stops all the threads when the specified elapsed time is passed. + * Before exiting, it prints the average execution for + * each operation and operation throughput. + */ + public int run(String[] args) throws Exception { + int exitCode = init(args); + if (exitCode != 0) { + return exitCode; + } + + barrier(); + + DFSClientThread[] threads = new DFSClientThread[numOfThreads]; + for (int i=0; i<numOfThreads; i++) { + threads[i] = new DFSClientThread(i); + threads[i].start(); + } + if (elapsedTime>0) { + Thread.sleep(elapsedTime*1000); + shouldRun = false; + } + for (DFSClientThread thread : threads) { + thread.join(); + for (int i=0; i<TOTAL_OP_TYPES; i++) { + executionTime[i] += thread.executionTime[i]; + totalNumOfOps[i] += thread.totalNumOfOps[i]; + } + } + long totalOps = 0; + for (int i=0; i<TOTAL_OP_TYPES; i++) { + totalOps += totalNumOfOps[i]; + } + + if (totalNumOfOps[OPEN] != 0) { + System.out.println("Average open execution time: " + + (double)executionTime[OPEN]/totalNumOfOps[OPEN] + "ms"); + } + if (totalNumOfOps[LIST] != 0) { + System.out.println("Average list execution time: " + + (double)executionTime[LIST]/totalNumOfOps[LIST] + "ms"); + } + if (totalNumOfOps[DELETE] != 0) { + System.out.println("Average deletion execution time: " + + (double)executionTime[DELETE]/totalNumOfOps[DELETE] + "ms"); + System.out.println("Average create execution time: " + + (double)executionTime[CREATE]/totalNumOfOps[CREATE] + "ms"); + System.out.println("Average write_close execution time: " + + (double)executionTime[WRITE_CLOSE]/totalNumOfOps[WRITE_CLOSE] + "ms"); + } + if (elapsedTime != 0) { + System.out.println("Average operations per second: " + + (double)totalOps/elapsedTime +"ops/s"); + } + System.out.println(); + return exitCode; + } + + /** Parse the command line arguments and initialize the data */ + private int init(String[] args) throws IOException { + try { + fs = FileSystem.get(getConf()); + } catch (IOException ioe) { + System.err.println("Can not initialize the file system: " + + ioe.getLocalizedMessage()); + return -1; + } + int hostHashCode = hostname.hashCode(); + try { + for (int i = 0; i < args.length; i++) { // parse command line + if (args[i].equals("-readProbability")) { + readPr = Double.parseDouble(args[++i]); + if (readPr<0 || readPr>1) { + System.err.println( + "The read probability must be [0, 1]: " + readPr); + return -1; + } + } else if (args[i].equals("-writeProbability")) { + writePr = Double.parseDouble(args[++i]); + if (writePr<0 || writePr>1) { + System.err.println( + "The write probability must be [0, 1]: " + writePr); + return -1; + } + } else if (args[i].equals("-root")) { + root = new Path(args[++i]); + } else if (args[i].equals("-maxDelayBetweenOps")) { + maxDelayBetweenOps = Integer.parseInt(args[++i]); // in milliseconds + } else if (args[i].equals("-numOfThreads")) { + numOfThreads = Integer.parseInt(args[++i]); + if (numOfThreads <= 0) { + System.err.println( + "Number of threads must be positive: " + numOfThreads); + return -1; + } + } else if (args[i].equals("-startTime")) { + startTime = Long.parseLong(args[++i]); + } else if (args[i].equals("-elapsedTime")) { + elapsedTime = Long.parseLong(args[++i]); + } else if (args[i].equals("-seed")) { + r = new Random(Long.parseLong(args[++i])+hostHashCode); + } else { + System.err.println(USAGE); + ToolRunner.printGenericCommandUsage(System.err); + return -1; + } + } + } catch (NumberFormatException e) { + System.err.println("Illegal parameter: " + e.getLocalizedMessage()); + System.err.println(USAGE); + return -1; + } + + if (readPr+writePr <0 || readPr+writePr>1) { + System.err.println( + "The sum of read probability and write probability must be [0, 1]: " + + readPr + " "+writePr); + return -1; + } + + if (r==null) { + r = new Random(System.currentTimeMillis()+hostHashCode); + } + + return initFileDirTables(); + } + + /** Create a table that contains all directories under root and + * another table that contains all files under root. + */ + private int initFileDirTables() { + try { + initFileDirTables(root); + } catch (IOException e) { + System.err.println(e.getLocalizedMessage()); + e.printStackTrace(); + return -1; + } + if (dirs.isEmpty()) { + System.err.println("The test space " + root + " is empty"); + return -1; + } + if (files.isEmpty()) { + System.err.println("The test space " + root + + " does not have any file"); + return -1; + } + return 0; + } + + /** Create a table that contains all directories under the specified path and + * another table that contains all files under the specified path and + * whose name starts with "_file_". + */ + private void initFileDirTables(Path path) throws IOException { + FileStatus[] stats = fs.listStatus(path); + if (stats != null) { + for (FileStatus stat : stats) { + if (stat.isDir()) { + dirs.add(stat.getPath().toString()); + initFileDirTables(stat.getPath()); + } else { + Path filePath = stat.getPath(); + if (filePath.getName().startsWith(StructureGenerator.FILE_NAME_PREFIX)) { + files.add(filePath.toString()); + } + } + } + } + } + + /** Returns when the current number of seconds from the epoch equals + * the command line argument given by <code>-startTime</code>. + * This allows multiple instances of this program, running on clock + * synchronized nodes, to start at roughly the same time. + */ + private void barrier() { + long sleepTime; + while ((sleepTime = startTime - System.currentTimeMillis()) > 0) { + try { + Thread.sleep(sleepTime); + } catch (InterruptedException ex) { + } + } + } + + /** Create a file with a length of <code>fileSize</code>. + * The file is filled with 'a'. + */ + private void genFile(Path file, long fileSize) throws IOException { + long startTime = System.currentTimeMillis(); + FSDataOutputStream out = fs.create(file, true, + getConf().getInt("io.file.buffer.size", 4096), + (short)getConf().getInt("dfs.replication", 3), + fs.getDefaultBlockSize()); + executionTime[CREATE] += (System.currentTimeMillis()-startTime); + totalNumOfOps[CREATE]++; + + for (long i=0; i<fileSize; i++) { + out.writeByte('a'); + } + startTime = System.currentTimeMillis(); + out.close(); + executionTime[WRITE_CLOSE] += (System.currentTimeMillis()-startTime); + totalNumOfOps[WRITE_CLOSE]++; + } + + /** Main program + * + * @param args command line arguments + * @throws Exception + */ + public static void main(String[] args) throws Exception { + int res = ToolRunner.run(new Configuration(), + new LoadGenerator(), args); + System.exit(res); + } + +}
Added: hadoop/core/trunk/src/test/org/apache/hadoop/fs/loadGenerator/StructureGenerator.java URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/test/org/apache/hadoop/fs/loadGenerator/StructureGenerator.java?rev=694458&view=auto ============================================================================== --- hadoop/core/trunk/src/test/org/apache/hadoop/fs/loadGenerator/StructureGenerator.java (added) +++ hadoop/core/trunk/src/test/org/apache/hadoop/fs/loadGenerator/StructureGenerator.java Thu Sep 11 13:23:09 2008 @@ -0,0 +1,307 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.loadGenerator; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.PrintStream; +import java.util.ArrayList; +import java.util.List; +import java.util.Random; + +import org.apache.hadoop.util.ToolRunner; + +/** + * This program generates a random namespace structure with the following + * constraints: + * 1. The number of subdirectories is a random number in [minWidth, maxWidth]. + * 2. The maximum depth of each subdirectory is a random number + * [2*maxDepth/3, maxDepth]. + * 3. Files are randomly placed in the empty directories. The size of each + * file follows Gaussian distribution. + * The generated namespace structure is described by two files in the output + * directory. Each line of the first file + * contains the full name of a leaf directory. + * Each line of the second file contains + * the full name of a file and its size, separated by a blank. + * + * The synopsis of the command is + * java StructureGenerator + -maxDepth <maxDepth> : maximum depth of the directory tree; default is 5. + -minWidth <minWidth> : minimum number of subdirectories per directories; default is 1 + -maxWidth <maxWidth> : maximum number of subdirectories per directories; default is 5 + -numOfFiles <#OfFiles> : the total number of files; default is 10. + -avgFileSize <avgFileSizeInBlocks>: average size of blocks; default is 1. + -outDir <outDir>: output directory; default is the current directory. + -seed <seed>: random number generator seed; default is the current time. + */ +public class StructureGenerator { + private int maxDepth = 5; + private int minWidth = 1; + private int maxWidth = 5; + private int numOfFiles = 10; + private double avgFileSize = 1; + private File outDir = DEFAULT_STRUCTURE_DIRECTORY; + final static private String USAGE = "java StructureGenerator\n" + + "-maxDepth <maxDepth>\n" + + "-minWidth <minWidth>\n" + + "-maxWidth <maxWidth>\n" + + "-numOfFiles <#OfFiles>\n" + + "-avgFileSize <avgFileSizeInBlocks>\n" + + "-outDir <outDir>\n" + + "-seed <seed>"; + + private Random r = null; + + /** Default directory for storing file/directory structure */ + final static File DEFAULT_STRUCTURE_DIRECTORY = new File("."); + /** The name of the file for storing directory structure */ + final static String DIR_STRUCTURE_FILE_NAME = "dirStructure"; + /** The name of the file for storing file structure */ + final static String FILE_STRUCTURE_FILE_NAME = "fileStructure"; + /** The name prefix for the files created by this program */ + final static String FILE_NAME_PREFIX = "_file_"; + + /** + * The main function first parses the command line arguments, + * then generates in-memory directory structure and outputs to a file, + * last generates in-memory files and outputs them to a file. + */ + public int run(String[] args) throws Exception { + int exitCode = 0; + exitCode = init(args); + if (exitCode != 0) { + return exitCode; + } + genDirStructure(); + output(new File(outDir, DIR_STRUCTURE_FILE_NAME)); + genFileStructure(); + outputFiles(new File(outDir, FILE_STRUCTURE_FILE_NAME)); + return exitCode; + } + + /** Parse the command line arguments and initialize the data */ + private int init(String[] args) { + try { + for (int i = 0; i < args.length; i++) { // parse command line + if (args[i].equals("-maxDepth")) { + maxDepth = Integer.parseInt(args[++i]); + if (maxDepth<1) { + System.err.println("maxDepth must be positive: " + maxDepth); + return -1; + } + } else if (args[i].equals("-minWidth")) { + minWidth = Integer.parseInt(args[++i]); + if (minWidth<0) { + System.err.println("minWidth must be positive: " + minWidth); + return -1; + } + } else if (args[i].equals("-maxWidth")) { + maxWidth = Integer.parseInt(args[++i]); + } else if (args[i].equals("-numOfFiles")) { + numOfFiles = Integer.parseInt(args[++i]); + if (numOfFiles<1) { + System.err.println("NumOfFiles must be positive: " + numOfFiles); + return -1; + } + } else if (args[i].equals("-avgFileSize")) { + avgFileSize = Double.parseDouble(args[++i]); + if (avgFileSize<=0) { + System.err.println("AvgFileSize must be positive: " + avgFileSize); + return -1; + } + } else if (args[i].equals("-outDir")) { + outDir = new File(args[++i]); + } else if (args[i].equals("-seed")) { + r = new Random(Long.parseLong(args[++i])); + } else { + System.err.println(USAGE); + ToolRunner.printGenericCommandUsage(System.err); + return -1; + } + } + } catch (NumberFormatException e) { + System.err.println("Illegal parameter: " + e.getLocalizedMessage()); + System.err.println(USAGE); + return -1; + } + + if (maxWidth < minWidth) { + System.err.println( + "maxWidth must be bigger than minWidth: " + maxWidth); + return -1; + } + + if (r==null) { + r = new Random(); + } + return 0; + } + + /** In memory representation of a directory */ + private static class INode { + private String name; + private List<INode> children = new ArrayList<INode>(); + + /** Constructor */ + private INode(String name) { + this.name = name; + } + + /** Add a child (subdir/file) */ + private void addChild(INode child) { + children.add(child); + } + + /** Output the subtree rooted at the current node. + * Only the leaves are printed. + */ + private void output(PrintStream out, String prefix) { + prefix = prefix==null?name:prefix+"/"+name; + if (children.isEmpty()) { + out.println(prefix); + } else { + for (INode child : children) { + child.output(out, prefix); + } + } + } + + /** Output the files in the subtree rooted at this node */ + protected void outputFiles(PrintStream out, String prefix) { + prefix = prefix==null?name:prefix+"/"+name; + for (INode child : children) { + child.outputFiles(out, prefix); + } + } + + /** Add all the leaves in the subtree to the input list */ + private void getLeaves(List<INode> leaves) { + if (children.isEmpty()) { + leaves.add(this); + } else { + for (INode child : children) { + child.getLeaves(leaves); + } + } + } + } + + /** In memory representation of a file */ + private static class FileINode extends INode { + private double numOfBlocks; + + /** constructor */ + private FileINode(String name, double numOfBlocks) { + super(name); + this.numOfBlocks = numOfBlocks; + } + + /** Output a file attribute */ + protected void outputFiles(PrintStream out, String prefix) { + prefix = (prefix == null)?super.name: prefix + "/"+super.name; + out.println(prefix + " " + numOfBlocks); + } + } + + private INode root; + + /** Generates a directory tree with a max depth of <code>maxDepth</code> */ + private void genDirStructure() { + root = genDirStructure("", maxDepth); + } + + /** Generate a directory tree rooted at <code>rootName</code> + * The number of subtree is in the range of [minWidth, maxWidth]. + * The maximum depth of each subtree is in the range of + * [2*maxDepth/3, maxDepth]. + */ + private INode genDirStructure(String rootName, int maxDepth) { + INode root = new INode(rootName); + + if (maxDepth>0) { + maxDepth--; + int minDepth = maxDepth*2/3; + // Figure out the number of subdirectories to generate + int numOfSubDirs = minWidth + r.nextInt(maxWidth-minWidth+1); + // Expand the tree + for (int i=0; i<numOfSubDirs; i++) { + int childDepth = (maxDepth == 0)?0: + (r.nextInt(maxDepth-minDepth+1)+minDepth); + INode child = genDirStructure("dir"+i, childDepth); + root.addChild(child); + } + } + return root; + } + + /** Collects leaf nodes in the tree */ + private List<INode> getLeaves() { + List<INode> leaveDirs = new ArrayList<INode>(); + root.getLeaves(leaveDirs); + return leaveDirs; + } + + /** Decides where to place all the files and its length. + * It first collects all empty directories in the tree. + * For each file, it randomly chooses an empty directory to place the file. + * The file's length is generated using Gaussian distribution. + */ + private void genFileStructure() { + List<INode> leaves = getLeaves(); + int totalLeaves = leaves.size(); + for (int i=0; i<numOfFiles; i++) { + int leaveNum = r.nextInt(totalLeaves); + double fileSize; + do { + fileSize = r.nextGaussian()+avgFileSize; + } while (fileSize<0); + leaves.get(leaveNum).addChild( + new FileINode(FILE_NAME_PREFIX+i, fileSize)); + } + } + + /** Output directory structure to a file, each line of the file + * contains the directory name. Only empty directory names are printed. */ + private void output(File outFile) throws FileNotFoundException { + System.out.println("Printing to " + outFile.toString()); + PrintStream out = new PrintStream(outFile); + root.output(out, null); + out.close(); + } + + /** Output all files' attributes to a file, each line of the output file + * contains a file name and its length. */ + private void outputFiles(File outFile) throws FileNotFoundException { + System.out.println("Printing to " + outFile.toString()); + PrintStream out = new PrintStream(outFile); + root.outputFiles(out, null); + out.close(); + } + + /** + * Main program + * @param args Command line arguments + * @throws Exception + */ + public static void main(String[] args) throws Exception { + StructureGenerator sg = new StructureGenerator(); + System.exit(sg.run(args)); + } +} Added: hadoop/core/trunk/src/test/org/apache/hadoop/fs/loadGenerator/TestLoadGenerator.java URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/test/org/apache/hadoop/fs/loadGenerator/TestLoadGenerator.java?rev=694458&view=auto ============================================================================== --- hadoop/core/trunk/src/test/org/apache/hadoop/fs/loadGenerator/TestLoadGenerator.java (added) +++ hadoop/core/trunk/src/test/org/apache/hadoop/fs/loadGenerator/TestLoadGenerator.java Thu Sep 11 13:23:09 2008 @@ -0,0 +1,217 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.fs.loadGenerator; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileReader; +import java.io.FileWriter; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hdfs.MiniDFSCluster; + +import junit.framework.TestCase; +/** + * This class tests if a balancer schedules tasks correctly. + */ +public class TestLoadGenerator extends TestCase { + private static final Configuration CONF = new Configuration(); + private static final int DEFAULT_BLOCK_SIZE = 10; + private static final String OUT_DIR = + System.getProperty("test.build.data","build/test/data"); + private static final File DIR_STRUCTURE_FILE = + new File(OUT_DIR, StructureGenerator.DIR_STRUCTURE_FILE_NAME); + private static final File FILE_STRUCTURE_FILE = + new File(OUT_DIR, StructureGenerator.FILE_STRUCTURE_FILE_NAME); + private static final String DIR_STRUCTURE_FIRST_LINE = "/dir0"; + private static final String DIR_STRUCTURE_SECOND_LINE = "/dir1"; + private static final String FILE_STRUCTURE_FIRST_LINE = + "/dir0/_file_0 0.3754598635933768"; + private static final String FILE_STRUCTURE_SECOND_LINE = + "/dir1/_file_1 1.4729310851145203"; + + + static { + CONF.setLong("dfs.block.size", DEFAULT_BLOCK_SIZE); + CONF.setInt("io.bytes.per.checksum", DEFAULT_BLOCK_SIZE); + CONF.setLong("dfs.heartbeat.interval", 1L); + } + + /** Test if the structure generator works fine */ + public void testStructureGenerator() throws Exception { + StructureGenerator sg = new StructureGenerator(); + String[] args = new String[]{"-maxDepth", "2", "-minWidth", "1", + "-maxWidth", "2", "-numOfFiles", "2", + "-avgFileSize", "1", "-outDir", OUT_DIR, "-seed", "1"}; + + final int MAX_DEPTH = 1; + final int MIN_WIDTH = 3; + final int MAX_WIDTH = 5; + final int NUM_OF_FILES = 7; + final int AVG_FILE_SIZE = 9; + final int SEED = 13; + try { + // successful case + assertEquals(0, sg.run(args)); + BufferedReader in = new BufferedReader(new FileReader(DIR_STRUCTURE_FILE)); + assertEquals(DIR_STRUCTURE_FIRST_LINE, in.readLine()); + assertEquals(DIR_STRUCTURE_SECOND_LINE, in.readLine()); + assertEquals(null, in.readLine()); + in.close(); + + in = new BufferedReader(new FileReader(FILE_STRUCTURE_FILE)); + assertEquals(FILE_STRUCTURE_FIRST_LINE, in.readLine()); + assertEquals(FILE_STRUCTURE_SECOND_LINE, in.readLine()); + assertEquals(null, in.readLine()); + in.close(); + + String oldArg = args[MAX_DEPTH]; + args[MAX_DEPTH] = "0"; + assertEquals(-1, sg.run(args)); + args[MAX_DEPTH] = oldArg; + + oldArg = args[MIN_WIDTH]; + args[MIN_WIDTH] = "-1"; + assertEquals(-1, sg.run(args)); + args[MIN_WIDTH] = oldArg; + + oldArg = args[MAX_WIDTH]; + args[MAX_WIDTH] = "-1"; + assertEquals(-1, sg.run(args)); + args[MAX_WIDTH] = oldArg; + + oldArg = args[NUM_OF_FILES]; + args[NUM_OF_FILES] = "-1"; + assertEquals(-1, sg.run(args)); + args[NUM_OF_FILES] = oldArg; + + oldArg = args[NUM_OF_FILES]; + args[NUM_OF_FILES] = "-1"; + assertEquals(-1, sg.run(args)); + args[NUM_OF_FILES] = oldArg; + + oldArg = args[AVG_FILE_SIZE]; + args[AVG_FILE_SIZE] = "-1"; + assertEquals(-1, sg.run(args)); + args[AVG_FILE_SIZE] = oldArg; + + oldArg = args[SEED]; + args[SEED] = "34.d4"; + assertEquals(-1, sg.run(args)); + args[SEED] = oldArg; + } finally { + DIR_STRUCTURE_FILE.delete(); + FILE_STRUCTURE_FILE.delete(); + } + } + + /** Test if the load generator works fine */ + public void testLoadGenerator() throws Exception { + final String TEST_SPACE_ROOT = "/test"; + + FileWriter writer = new FileWriter(DIR_STRUCTURE_FILE); + writer.write(DIR_STRUCTURE_FIRST_LINE+"\n"); + writer.write(DIR_STRUCTURE_SECOND_LINE+"\n"); + writer.close(); + + writer = new FileWriter(FILE_STRUCTURE_FILE); + writer.write(FILE_STRUCTURE_FIRST_LINE+"\n"); + writer.write(FILE_STRUCTURE_SECOND_LINE+"\n"); + writer.close(); + + MiniDFSCluster cluster = new MiniDFSCluster(CONF, 3, true, null); + cluster.waitActive(); + + try { + DataGenerator dg = new DataGenerator(); + dg.setConf(CONF); + String [] args = new String[] {"-inDir", OUT_DIR, "-root", TEST_SPACE_ROOT}; + assertEquals(0, dg.run(args)); + + final int READ_PROBABILITY = 1; + final int WRITE_PROBABILITY = 3; + final int MAX_DELAY_BETWEEN_OPS = 7; + final int NUM_OF_THREADS = 9; + final int START_TIME = 11; + final int ELAPSED_TIME = 13; + + LoadGenerator lg = new LoadGenerator(); + lg.setConf(CONF); + args = new String[] {"-readProbability", "0.3", "-writeProbability", "0.3", + "-root", TEST_SPACE_ROOT, "-maxDelayBetweenOps", "0", + "-numOfThreads", "1", "-startTime", + Long.toString(System.currentTimeMillis()), "-elapsedTime", "10"}; + + assertEquals(0, lg.run(args)); + + String oldArg = args[READ_PROBABILITY]; + args[READ_PROBABILITY] = "1.1"; + assertEquals(-1, lg.run(args)); + args[READ_PROBABILITY] = "-1.1"; + assertEquals(-1, lg.run(args)); + args[READ_PROBABILITY] = oldArg; + + oldArg = args[WRITE_PROBABILITY]; + args[WRITE_PROBABILITY] = "1.1"; + assertEquals(-1, lg.run(args)); + args[WRITE_PROBABILITY] = "-1.1"; + assertEquals(-1, lg.run(args)); + args[WRITE_PROBABILITY] = "0.9"; + assertEquals(-1, lg.run(args)); + args[READ_PROBABILITY] = oldArg; + + oldArg = args[MAX_DELAY_BETWEEN_OPS]; + args[MAX_DELAY_BETWEEN_OPS] = "1.x1"; + assertEquals(-1, lg.run(args)); + args[MAX_DELAY_BETWEEN_OPS] = oldArg; + + oldArg = args[MAX_DELAY_BETWEEN_OPS]; + args[MAX_DELAY_BETWEEN_OPS] = "1.x1"; + assertEquals(-1, lg.run(args)); + args[MAX_DELAY_BETWEEN_OPS] = oldArg; + + oldArg = args[NUM_OF_THREADS]; + args[NUM_OF_THREADS] = "-1"; + assertEquals(-1, lg.run(args)); + args[NUM_OF_THREADS] = oldArg; + + oldArg = args[START_TIME]; + args[START_TIME] = "-1"; + assertEquals(-1, lg.run(args)); + args[START_TIME] = oldArg; + + oldArg = args[ELAPSED_TIME]; + args[ELAPSED_TIME] = "-1"; + assertEquals(-1, lg.run(args)); + args[ELAPSED_TIME] = oldArg; + } finally { + cluster.shutdown(); + DIR_STRUCTURE_FILE.delete(); + FILE_STRUCTURE_FILE.delete(); + } + } + + /** + * @param args + */ + public static void main(String[] args) throws Exception { + TestLoadGenerator loadGeneratorTest = new TestLoadGenerator(); + loadGeneratorTest.testStructureGenerator(); + loadGeneratorTest.testLoadGenerator(); + } +}
