Author: mahadev
Date: Tue Jul 22 11:23:28 2008
New Revision: 678845
URL: http://svn.apache.org/viewvc?rev=678845&view=rev
Log:
HADOOP-3624. Improving createeditslog to create tree directory structure.
(Lohit Vijayarenu via mahadev)
Added:
hadoop/core/trunk/src/test/org/apache/hadoop/hdfs/FileNameGenerator.java
Modified:
hadoop/core/trunk/CHANGES.txt
hadoop/core/trunk/src/test/org/apache/hadoop/hdfs/CreateEditsLog.java
hadoop/core/trunk/src/test/org/apache/hadoop/hdfs/NNThroughputBenchmark.java
Modified: hadoop/core/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/core/trunk/CHANGES.txt?rev=678845&r1=678844&r2=678845&view=diff
==============================================================================
--- hadoop/core/trunk/CHANGES.txt (original)
+++ hadoop/core/trunk/CHANGES.txt Tue Jul 22 11:23:28 2008
@@ -155,6 +155,9 @@
HADOOP-3795. Fix saving image files on Namenode with different checkpoint
stamps. (Lohit Vijayarenu via mahadev)
+
+ HADOOP-3624. Improving createeditslog to create tree directory structure.
+ (Lohit Vijayarenu via mahadev)
Release 0.18.0 - Unreleased
Modified: hadoop/core/trunk/src/test/org/apache/hadoop/hdfs/CreateEditsLog.java
URL:
http://svn.apache.org/viewvc/hadoop/core/trunk/src/test/org/apache/hadoop/hdfs/CreateEditsLog.java?rev=678845&r1=678844&r2=678845&view=diff
==============================================================================
--- hadoop/core/trunk/src/test/org/apache/hadoop/hdfs/CreateEditsLog.java
(original)
+++ hadoop/core/trunk/src/test/org/apache/hadoop/hdfs/CreateEditsLog.java Tue
Jul 22 11:23:28 2008
@@ -56,14 +56,12 @@
public class CreateEditsLog {
static final String BASE_PATH = "/createdViaInjectingInEditsLog";
- static final String BASE_PATH_PLUS_FILEPREFIX = BASE_PATH + "/file";
-
static final String EDITS_DIR = "/tmp/EditsLogOut";
-
static String edits_dir = EDITS_DIR;
static void addFiles(FSEditLog editLog, int numFiles, short replication,
- int blocksPerFile, long startingBlockId) {
+ int blocksPerFile, long startingBlockId,
+ FileNameGenerator nameGenerator) {
PermissionStatus p = new PermissionStatus("joeDoe", "people",
new FsPermission((short)0777));
@@ -80,7 +78,7 @@
long currentBlockId = startingBlockId;
long bidAtSync = startingBlockId;
- for (int iF = 1; iF <= numFiles; iF++) {
+ for (int iF = 0; iF < numFiles; iF++) {
for (int iB = 0; iB < blocksPerFile; ++iB) {
blocks[iB].setBlockId(currentBlockId++);
}
@@ -89,11 +87,16 @@
INodeFileUnderConstruction inode = new INodeFileUnderConstruction(
null, replication, 0, blockSize, blocks, p, "", "",
null);
- String path = BASE_PATH_PLUS_FILEPREFIX + iF +
- "_B" + blocks[0].getBlockId() +
- "_to_B" + blocks[blocksPerFile-1].getBlockId();
- editLog.logOpenFile(path, inode);
- editLog.logCloseFile(path, inode);
+ String path = iF + "_B" + blocks[0].getBlockId() +
+ "_to_B" + blocks[blocksPerFile-1].getBlockId() + "_";
+ String filePath = nameGenerator.getNextFileName(path);
+ if ((iF % nameGenerator.getFilesPerDirectory()) == 0) {
+ String currentDir = nameGenerator.getCurrentDir();
+ dirInode = new INodeDirectory(p, 0L);
+ editLog.logMkDir(currentDir, dirInode);
+ }
+ editLog.logOpenFile(filePath, inode);
+ editLog.logCloseFile(filePath, inode);
if (currentBlockId - bidAtSync >= 2000) { // sync every 2K blocks
editLog.logSync();
@@ -145,6 +148,8 @@
}
for (int i = 0; i < args.length; i++) { // parse command line
+ if (args[i].equals("-h"))
+ printUsageExit();
if (args[i].equals("-f")) {
if (i + 3 >= args.length || args[i+1].startsWith("-") ||
args[i+2].startsWith("-") || args[i+3].startsWith("-")) {
@@ -191,12 +196,14 @@
}
FSImage fsImage = new FSImage(new File(edits_dir));
+ FileNameGenerator nameGenerator = new FileNameGenerator(BASE_PATH, 100);
FSEditLog editLog = fsImage.getEditLog();
editLog.createEditLogFile(fsImage.getFsEditName());
editLog.open();
- addFiles(editLog, numFiles, replication, numBlocksPerFile,
startingBlockId);
+ addFiles(editLog, numFiles, replication, numBlocksPerFile, startingBlockId,
+ nameGenerator);
editLog.logSync();
editLog.close();
}
Added: hadoop/core/trunk/src/test/org/apache/hadoop/hdfs/FileNameGenerator.java
URL:
http://svn.apache.org/viewvc/hadoop/core/trunk/src/test/org/apache/hadoop/hdfs/FileNameGenerator.java?rev=678845&view=auto
==============================================================================
--- hadoop/core/trunk/src/test/org/apache/hadoop/hdfs/FileNameGenerator.java
(added)
+++ hadoop/core/trunk/src/test/org/apache/hadoop/hdfs/FileNameGenerator.java
Tue Jul 22 11:23:28 2008
@@ -0,0 +1,93 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs;
+
+import java.util.Arrays;
+
+/**
+ * File name generator.
+ *
+ * Each directory contains not more than a fixed number (filesPerDir)
+ * of files and directories.
+ * When the number of files in one directory reaches the maximum,
+ * the generator creates a new directory and proceeds generating files in it.
+ * The generated namespace tree is balanced that is any path to a leaf
+ * file is not less than the height of the tree minus one.
+ */
+public class FileNameGenerator {
+ private static final int DEFAULT_FILES_PER_DIRECTORY = 32;
+
+ private int[] pathIndecies = new int[20]; // this will support up to 32**20
= 2**100 = 10**30 files
+ private String baseDir;
+ private String currentDir;
+ private int filesPerDirectory;
+ private long fileCount;
+
+ FileNameGenerator(String baseDir) {
+ this(baseDir, DEFAULT_FILES_PER_DIRECTORY);
+ }
+
+ FileNameGenerator(String baseDir, int filesPerDir) {
+ this.baseDir = baseDir;
+ this.filesPerDirectory = filesPerDir;
+ reset();
+ }
+
+ String getNextDirName(String prefix) {
+ int depth = 0;
+ while(pathIndecies[depth] >= 0)
+ depth++;
+ int level;
+ for(level = depth-1;
+ level >= 0 && pathIndecies[level] == filesPerDirectory-1; level--)
+ pathIndecies[level] = 0;
+ if(level < 0)
+ pathIndecies[depth] = 0;
+ else
+ pathIndecies[level]++;
+ level = 0;
+ String next = baseDir;
+ while(pathIndecies[level] >= 0)
+ next = next + "/" + prefix + pathIndecies[level++];
+ return next;
+ }
+
+ synchronized String getNextFileName(String fileNamePrefix) {
+ long fNum = fileCount % filesPerDirectory;
+ if(fNum == 0) {
+ currentDir = getNextDirName(fileNamePrefix + "Dir");
+ }
+ String fn = currentDir + "/" + fileNamePrefix + fileCount;
+ fileCount++;
+ return fn;
+ }
+
+ private synchronized void reset() {
+ Arrays.fill(pathIndecies, -1);
+ fileCount = 0L;
+ currentDir = "";
+ }
+
+ synchronized int getFilesPerDirectory() {
+ return filesPerDirectory;
+ }
+
+ synchronized String getCurrentDir() {
+ return currentDir;
+ }
+}
Modified:
hadoop/core/trunk/src/test/org/apache/hadoop/hdfs/NNThroughputBenchmark.java
URL:
http://svn.apache.org/viewvc/hadoop/core/trunk/src/test/org/apache/hadoop/hdfs/NNThroughputBenchmark.java?rev=678845&r1=678844&r2=678845&view=diff
==============================================================================
---
hadoop/core/trunk/src/test/org/apache/hadoop/hdfs/NNThroughputBenchmark.java
(original)
+++
hadoop/core/trunk/src/test/org/apache/hadoop/hdfs/NNThroughputBenchmark.java
Tue Jul 22 11:23:28 2008
@@ -365,71 +365,6 @@
}
/**
- * File name generator.
- *
- * Each directory contains not more than a fixed number (filesPerDir)
- * of files and directories.
- * When the number of files in one directory reaches the maximum,
- * the generator creates a new directory and proceeds generating files in it.
- * The generated namespace tree is balanced that is any path to a leaf
- * file is not less than the height of the tree minus one.
- */
- private static class FileGenerator {
- private static final int DEFAULT_FILES_PER_DIRECTORY = 32;
- // Average file name size is 16.5 bytes
- private static final String FILE_NAME_PREFFIX ="ThrouputBenchfile"; // 17
bytes
- private static final String DIR_NAME_PREFFIX = "ThrouputBenchDir"; // 16
bytes
- // private static final int NUM_CLIENTS = 100;
-
- private int[] pathIndecies = new int[20]; // this will support up to
32**20 = 2**100 = 10**30 files
- private String baseDir;
- private String currentDir;
- private int filesPerDirectory = DEFAULT_FILES_PER_DIRECTORY;
- private long fileCount;
-
- FileGenerator(String baseDir, int filesPerDir) {
- this.baseDir = baseDir;
- this.filesPerDirectory = filesPerDir;
- reset();
- }
-
- String getNextDirName() {
- int depth = 0;
- while(pathIndecies[depth] >= 0)
- depth++;
- int level;
- for(level = depth-1;
- level >= 0 && pathIndecies[level] == filesPerDirectory-1; level--)
- pathIndecies[level] = 0;
- if(level < 0)
- pathIndecies[depth] = 0;
- else
- pathIndecies[level]++;
- level = 0;
- String next = baseDir;
- while(pathIndecies[level] >= 0)
- next = next + "/" + DIR_NAME_PREFFIX + pathIndecies[level++];
- return next;
- }
-
- synchronized String getNextFileName() {
- long fNum = fileCount % filesPerDirectory;
- if(fNum == 0) {
- currentDir = getNextDirName();
- }
- String fn = currentDir + "/" + FILE_NAME_PREFFIX + fileCount;
- fileCount++;
- return fn;
- }
-
- private synchronized void reset() {
- Arrays.fill(pathIndecies, -1);
- fileCount = 0L;
- currentDir = "";
- }
- }
-
- /**
* File creation statistics.
*
* Each thread creates the same (+ or -1) number of files.
@@ -442,7 +377,7 @@
static final String OP_CREATE_USAGE =
"-op create [-threads T] [-files N] [-filesPerDir P]";
- protected FileGenerator nameGenerator;
+ protected FileNameGenerator nameGenerator;
protected String[][] fileNames;
CreateFileStats(String[] args) {
@@ -470,7 +405,7 @@
} else if(!ignoreUnrelatedOptions)
printUsage();
}
- nameGenerator = new FileGenerator(getBaseDir(), nrFilesPerDir);
+ nameGenerator = new FileNameGenerator(getBaseDir(), nrFilesPerDir);
}
void generateInputs(int[] opsPerThread) throws IOException {
@@ -483,7 +418,8 @@
int threadOps = opsPerThread[idx];
fileNames[idx] = new String[threadOps];
for(int jdx=0; jdx < threadOps; jdx++)
- fileNames[idx][jdx] = nameGenerator.getNextFileName();
+ fileNames[idx][jdx] = nameGenerator.
+ getNextFileName("ThroughputBench");
}
}
@@ -516,7 +452,7 @@
LOG.info("--- " + getOpName() + " inputs ---");
LOG.info("nrFiles = " + numOpsRequired);
LOG.info("nrThreads = " + numThreads);
- LOG.info("nrFilesPerDir = " + nameGenerator.filesPerDirectory);
+ LOG.info("nrFilesPerDir = " + nameGenerator.getFilesPerDirectory());
printStats();
}
}
@@ -548,7 +484,8 @@
"-op", "create",
"-threads", String.valueOf(this.numThreads),
"-files", String.valueOf(numOpsRequired),
- "-filesPerDir", String.valueOf(nameGenerator.filesPerDirectory)};
+ "-filesPerDir",
+ String.valueOf(nameGenerator.getFilesPerDirectory())};
CreateFileStats opCreate = new CreateFileStats(createArgs);
opCreate.benchmark();
LOG.info("Created " + numOpsRequired + " files.");
@@ -782,12 +719,12 @@
// create files
LOG.info("Creating " + nrFiles + " with " + blocksPerFile + " blocks
each.");
- FileGenerator nameGenerator;
- nameGenerator = new FileGenerator(getBaseDir(), 100);
+ FileNameGenerator nameGenerator;
+ nameGenerator = new FileNameGenerator(getBaseDir(), 100);
String clientName = getClientName(007);
nameNode.setSafeMode(FSConstants.SafeModeAction.SAFEMODE_LEAVE);
for(int idx=0; idx < nrFiles; idx++) {
- String fileName = nameGenerator.getNextFileName();
+ String fileName = nameGenerator.getNextFileName("ThroughputBench");
nameNode.create(fileName, FsPermission.getDefault(),
clientName, true, replication, BLOCK_SIZE);
addBlocks(fileName, clientName);
