svn commit: r604138 - in /lucene/hadoop/site: author/src/documentation/content/xdocs/credits.xml publish/credits.html publish/credits.pdf
Author: shv Date: Fri Dec 14 01:31:47 2007 New Revision: 604138 URL: http://svn.apache.org/viewvc?rev=604138view=rev Log: Add Konstantin Shvachko to list of committers. Modified: lucene/hadoop/site/author/src/documentation/content/xdocs/credits.xml lucene/hadoop/site/publish/credits.html lucene/hadoop/site/publish/credits.pdf Modified: lucene/hadoop/site/author/src/documentation/content/xdocs/credits.xml URL: http://svn.apache.org/viewvc/lucene/hadoop/site/author/src/documentation/content/xdocs/credits.xml?rev=604138r1=604137r2=604138view=diff == --- lucene/hadoop/site/author/src/documentation/content/xdocs/credits.xml (original) +++ lucene/hadoop/site/author/src/documentation/content/xdocs/credits.xml Fri Dec 14 01:31:47 2007 @@ -97,7 +97,15 @@ td/td td-8/td /tr - + + tr +tdshv/td +tda href=http://people.apache.org/~shv;Konstantin Shvachko/a/td +tdYahoo!/td +tdHDFS/td +td-8/td + /tr + tr tdstack/td tdMichael Stack/td Modified: lucene/hadoop/site/publish/credits.html URL: http://svn.apache.org/viewvc/lucene/hadoop/site/publish/credits.html?rev=604138r1=604137r2=604138view=diff == --- lucene/hadoop/site/publish/credits.html (original) +++ lucene/hadoop/site/publish/credits.html Fri Dec 14 01:31:47 2007 @@ -294,7 +294,18 @@ td colspan=1 rowspan=1-8/td /tr + + +tr +td colspan=1 rowspan=1shv/td +td colspan=1 rowspan=1a href=http://people.apache.org/~shv;Konstantin Shvachko/a/td +td colspan=1 rowspan=1Yahoo!/td +td colspan=1 rowspan=1HDFS/td +td colspan=1 rowspan=1-8/td + +/tr + tr @@ -333,7 +344,7 @@ /div -a name=N1018B/aa name=Contributors/a +a name=N101A9/aa name=Contributors/a h2 class=h3Contributors/h2 div class=section pA list of Hadoop contributors and their contributions is available from @@ -343,7 +354,7 @@ /div -a name=N10199/aa name=Emeriti/a +a name=N101B7/aa name=Emeriti/a h2 class=h3Emeriti/h2 div class=section pCommitters who are no longer active on Hadoop are:/p Modified: lucene/hadoop/site/publish/credits.pdf URL: http://svn.apache.org/viewvc/lucene/hadoop/site/publish/credits.pdf?rev=604138r1=604137r2=604138view=diff == --- lucene/hadoop/site/publish/credits.pdf (original) +++ lucene/hadoop/site/publish/credits.pdf Fri Dec 14 01:31:47 2007 @@ -58,10 +58,10 @@ endobj 14 0 obj - /Length 3437 /Filter [ /ASCII85Decode /FlateDecode ] + /Length 3651 /Filter [ /ASCII85Decode /FlateDecode ] stream -GatV#?'!_u(5D;@=(nek,)@[EMAIL PROTECTED]@NF?=;:./BP'*0'/[EMAIL PROTECTED]/6nJba/e6QUZ;*F.7^RZYC:lB,Eg)=FSgJ7%5\-1YCU'[EMAIL PROTECTED](\DR5JJT9(/[EMAIL PROTECTED]Q8Z!]-*LPV:raQd[D([EMAIL PROTECTED];.[EMAIL PROTECTED]VBjHt4\Z_TkU+44.;[EMAIL PROTECTED])eN_kP#s#+.:s[V]ePK0dqI3JsP,OVDeMAiP^^N$.[U3W.8b`PHh/^h-OC`VE;U`7u'93JneoJ0o(hf-M+BB'SusW,e2`RlP/N($fCj[EMAIL PROTECTED];SE^On=)4%W8O;gcfP#Paa)\iKSb0cRr+cD`7G*#hPs[84*cj6We?k_9:k((/\iGF_AIORA[I3#?%;Sq(a(qXN4u$1L+e'X,\u]p'^$3C4KAU!';N(+]W0b'%L1?6m7qs%sW'%LERi!F8$`Q*3SYn2P(eKJfW-n5^7m5)kP\L3AUHuo$0\CTlr-3L3#Z4WPo62;?Y(2btVGN\NW-_?`R!$p4/'MHgn$fGLcZTg-s1.H8+H8dm=kS[0qPP)[EMAIL PROTECTED]')Mc,E#Uc$?)a,?A9LqA!QE-.[]/291E!UW,2!kTVR(ZS4pk@:h14ablrYqEqb:A42V/5'm,.gaihBCaH#b5;.:mC%Kt81b$:tX?Wmqi;?YXNnH5BHpNP#MQ[EMAIL PROTECTED]d'_:[EMAIL PROTECTED]:ACoV.aH9/`g7m;VatbUP$*[QL^ODXO^+^W_:u+RfIaT\6[MV][EMAIL PROTECTED])gM6JgZ1uRV.)).66TECFdb\LLN$ii`7*mQ5VLHQAVh@':[EMAIL PROTECTED] O1q;S^qI1I*hES,YlBC_#:7ulGI5Mn1\*tp@@U)C8c7'EL$3_ioN%1HO9OIZhYPOe%[.4!`3IgKD,UUCCi2)iDj*N^VA(gD/#pqT.JS(K4rbc[G1!i_3h135L%[EMAIL PROTECTED]-5dMSdb0uRJ@/djIghX[)aiJe8/YVtLe1)dWldS!Xl+ga.B1H1Gg?JN4!G_!q,Nt8gY/FIn9p[D!?mhn,b(hb4'.d/@ig)rA4!#L]]aBa.4rgHYWD]l^2AGj%8;RY[7mroF\%CrF=MJ)oW0.*pDb6e$+'_+tQHM#0bgkMZ%N,Is$+M#=')FQU47E\*?8BpZ3bg`qOD=c5c(r;D%4tSQD_3p=m^3dC00q:[EMAIL PROTECTED]b.b'u]+Lp/[EMAIL PROTECTED]i%ntMMs-Qfeal)n:\'+[EMAIL PROTECTED]:48oZO1dZu!=R.fNTPW?UM7WqMma=W-I:u'V;Y65,`QNLqNWO6CCV[f-Y1^C2CJaT?-\;Muta-uZL:f3[7O0eUt4gcKl+FZ+\696eE90A$15,,!b7b`s7a(Xp3Ff6+bm3jAK;-R8$Z2*2`D[*[c^Oj`uRm^SN/J8a0o`c#IJXWns(JUU=Dl(,t?;=mfn'?OFN8-Y6FONS^)MroZ^36EW]:T6NhOYmIf$+MWc0(BSaik]*7DEZerCNStFXR%(n]TW?UHN8Vf#T,66V:[EMAIL PROTECTED]73:ccRR,.6Uot-@+jYI.Q)8\8oV6R-474[/'V[$Zlm51TPV$Z:QI!n?XVYVD1Jtidd2$,AH!g$*W*UOh6(Mej^CGf0rT]([^L4Y4:A8Le;HaM[EMAIL PROTECTED]/nBn#au/--Y!s3IX5!8S\L;/kSR-T_.:oHC,2NUd-\=N(l'#C1EXQl,^!E/JY$i(i[EMAIL PROTECTED])Io]K[;T oM*BL?709ZDkHRiE'[EMAIL PROTECTED](O;Ihs5a'[EMAIL PROTECTED]n^60\t[;J\jO2hc1%K(IGMs:uBl2?*1uroEOE9upc3h^U='V:u7YH:hu0nLbTb)#a2'?tII$\j;MsW-uGKE;E25`Mg2K_:986qO#W!4Nm.uI_KfWp@'INufB*$V9/[:We64!6VCS9d_4V1dU4bc\!ba]^6hnGfl$kADq@/,R$dpP^:fbCG4R)QUc_97(E\.^reO'k-biMuAsVn90FOXFZB6mKW=eJaB[+!0VMLZ=bPJrYK'@ccaeS6[*n77UkbKN[7,mUi)D$AWf#63jkHEh6rek(GEI#t^BK-CG9/h3gM2%/GkB6Z/[EMAIL
svn commit: r606744 [1/3] - in /lucene/hadoop/trunk: CHANGES.txt src/java/org/apache/hadoop/dfs/FSNamesystem.java src/test/org/apache/hadoop/dfs/NNThroughputBenchmark.java src/test/org/apache/hadoop/d
Author: shv Date: Mon Dec 24 14:20:52 2007 New Revision: 606744 URL: http://svn.apache.org/viewvc?rev=606744view=rev Log: HADOOP-2149. Name-node benchmark to evaluate its performance for optimizations and new features. (Konstantin Shvachko via shv) Added: lucene/hadoop/trunk/src/test/org/apache/hadoop/dfs/NNThroughputBenchmark.java lucene/hadoop/trunk/src/test/org/apache/hadoop/dfs/TestNNThroughputBenchmark.java Modified: lucene/hadoop/trunk/CHANGES.txt lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/FSNamesystem.java Modified: lucene/hadoop/trunk/CHANGES.txt URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/CHANGES.txt?rev=606744r1=606743r2=606744view=diff == --- lucene/hadoop/trunk/CHANGES.txt (original) +++ lucene/hadoop/trunk/CHANGES.txt Mon Dec 24 14:20:52 2007 @@ -132,6 +132,10 @@ HADOOP-2457. Added a 'forrest.home' property to the 'docs' target in build.xml. (acmurthy) +HADOOP-2149. A new benchmark for three name-node operation: file create, +open, and block report, to evaluate the name-node performance +for optimizations or new features. (Konstantin Shvachko via shv) + OPTIMIZATIONS HADOOP-1898. Release the lock protecting the last time of the last stack
svn commit: r606744 [3/3] - in /lucene/hadoop/trunk: CHANGES.txt src/java/org/apache/hadoop/dfs/FSNamesystem.java src/test/org/apache/hadoop/dfs/NNThroughputBenchmark.java src/test/org/apache/hadoop/d
Added:
lucene/hadoop/trunk/src/test/org/apache/hadoop/dfs/NNThroughputBenchmark.java
URL:
http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/test/org/apache/hadoop/dfs/NNThroughputBenchmark.java?rev=606744view=auto
==
---
lucene/hadoop/trunk/src/test/org/apache/hadoop/dfs/NNThroughputBenchmark.java
(added)
+++
lucene/hadoop/trunk/src/test/org/apache/hadoop/dfs/NNThroughputBenchmark.java
Mon Dec 24 14:20:52 2007
@@ -0,0 +1,787 @@
+package org.apache.hadoop.dfs;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
+import java.util.ArrayList;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.commons.logging.impl.Log4JLogger;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.net.DNS;
+import org.apache.hadoop.net.NetworkTopology;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.log4j.Level;
+
+/**
+ * Main class for a series of name-node benchmarks.
+ *
+ * Each benchmark measures throughput and average execution time
+ * of a specific name-node operation, e.g. file creation or block reports.
+ *
+ * The benchmark does not involve any other hadoop components
+ * except for the name-node. Each operation is executed
+ * by calling directly the respective name-node method.
+ * The name-node here is real all other components are simulated.
+ *
+ * Command line arguments for the benchmark include:br
+ * 1) total number of operations to be performed,br
+ * 2) number of threads to run these operations,br
+ * 3) followed by operation specific input parameters.
+ *
+ * Then the benchmark generates inputs for each thread so that the
+ * input generation overhead does not effect the resulting statistics.
+ * The number of operations performed by threads practically is the same.
+ * Precisely, the difference between the number of operations
+ * performed by any two threads does not exceed 1.
+ *
+ * Then the benchmark executes the specified number of operations using
+ * the specified number of threads and outputs the resulting stats.
+ */
+public class NNThroughputBenchmark {
+ private static final Log LOG =
LogFactory.getLog(org.apache.hadoop.dfs.NNThroughputBenchmark);
+ private static final int BLOCK_SIZE = 16;
+
+ static Configuration config;
+ static NameNode nameNode;
+
+ NNThroughputBenchmark(Configuration conf) throws IOException {
+config = conf;
+// We do not need many handlers, since each thread simulates a handler
+// by calling name-node methods directly
+config.setInt(dfs.namenode.handler.count, 1);
+// Start the NameNode
+String[] args = new String[] {};
+nameNode = NameNode.createNameNode(args, config);
+ }
+
+ void close() throws IOException {
+nameNode.stop();
+ }
+
+ static void turnOffNameNodeLogging() {
+// change log level to ERROR: NameNode.LOG NameNode.stateChangeLog
+((Log4JLogger)NameNode.LOG).getLogger().setLevel(Level.ERROR);
+((Log4JLogger)NameNode.stateChangeLog).getLogger().setLevel(Level.ERROR);
+((Log4JLogger)NetworkTopology.LOG).getLogger().setLevel(Level.ERROR);
+((Log4JLogger)FSNamesystem.LOG).getLogger().setLevel(Level.ERROR);
+ }
+
+ /**
+ * Base class for collecting operation statistics.
+ *
+ * Overload this class in order to run statistics for a
+ * specific name-node operation.
+ */
+ abstract class OperationStatsBase {
+protected static final String BASE_DIR_NAME = /nnThroughputBenchmark;
+protected static final String OP_ALL_NAME = all;
+protected static final String OP_ALL_USAGE = -op all other ops options;
+
+protected String baseDir;
+protected short replication;
+protected int numThreads = 0;// number of threads
+protected int numOpsRequired = 0;// number of operations requested
+protected int numOpsExecuted = 0;// number of operations executed
+protected long cumulativeTime = 0;// sum of times for each op
+protected long elapsedTime = 0; // time from start to finish
+
+/**
+ * Operation name.
+ */
+abstract String getOpName();
+
+/**
+ * Parse command line arguments.
+ *
+ * @param args arguments
+ * @throws IOException
+ */
+abstract void parseArguments(String[] args) throws IOException;
+
+/**
+ * Generate inputs for each daemon thread.
+ *
+ * @param opsPerThread number of inputs for each thread.
+ * @throws IOException
+ */
+abstract void generateInputs(int[] opsPerThread) throws IOException;
+
+/**
+ * This corresponds to the arg1 argument of
+ * [EMAIL PROTECTED] #executeOp(int, int, String)}, which can have
different meanings
+ * depending on the operation performed.
+ *
+ * @param daemonId
+ * @return the argument
+ */
+abstract String getExecutionArgument(int daemonId);
+
+/**
+ * Execute
svn commit: r607126 [1/3] - in /lucene/hadoop/trunk/src: java/org/apache/hadoop/dfs/FSNamesystem.java test/org/apache/hadoop/dfs/NNThroughputBenchmark.java test/org/apache/hadoop/dfs/TestNNThroughputB
Author: shv Date: Thu Dec 27 13:39:12 2007 New Revision: 607126 URL: http://svn.apache.org/viewvc?rev=607126view=rev Log: Line breaks corrected to unix style vs windows introduced by HADOOP-2149. Contributed by Konstantin Shvachko Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/FSNamesystem.java lucene/hadoop/trunk/src/test/org/apache/hadoop/dfs/NNThroughputBenchmark.java lucene/hadoop/trunk/src/test/org/apache/hadoop/dfs/TestNNThroughputBenchmark.java
svn commit: r612000 - in /lucene/hadoop/trunk: CHANGES.txt src/test/org/apache/hadoop/dfs/TestNNThroughputBenchmark.java
Author: shv
Date: Mon Jan 14 19:13:16 2008
New Revision: 612000
URL: http://svn.apache.org/viewvc?rev=612000view=rev
Log:
HADOOP-2601. Start name-node on a free port for TestNNThroughputBenchmark.
Contributed by Konstantin Shvachko
Modified:
lucene/hadoop/trunk/CHANGES.txt
lucene/hadoop/trunk/src/test/org/apache/hadoop/dfs/TestNNThroughputBenchmark.java
Modified: lucene/hadoop/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/lucene/hadoop/trunk/CHANGES.txt?rev=612000r1=611999r2=612000view=diff
==
--- lucene/hadoop/trunk/CHANGES.txt (original)
+++ lucene/hadoop/trunk/CHANGES.txt Mon Jan 14 19:13:16 2008
@@ -445,6 +445,9 @@
HADOOP-2481. NNBench report its progress periodically.
(Hairong Kuang via dhruba)
+ HADOOP-2601. Start name-node on a free port for
TestNNThroughputBenchmark.
+ (Konstantin Shvachko)
+
Release 0.15.3 - (unreleased changes)
BUG FIXES
Modified:
lucene/hadoop/trunk/src/test/org/apache/hadoop/dfs/TestNNThroughputBenchmark.java
URL:
http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/test/org/apache/hadoop/dfs/TestNNThroughputBenchmark.java?rev=612000r1=611999r2=612000view=diff
==
---
lucene/hadoop/trunk/src/test/org/apache/hadoop/dfs/TestNNThroughputBenchmark.java
(original)
+++
lucene/hadoop/trunk/src/test/org/apache/hadoop/dfs/TestNNThroughputBenchmark.java
Mon Jan 14 19:13:16 2008
@@ -10,7 +10,7 @@
*/
public void testNNThroughput() throws Exception {
Configuration conf = new Configuration();
-conf.set(fs.default.name, localhost:+Integer.toString(50017));
+conf.set(fs.default.name, localhost: + 0);
NameNode.format(conf);
NNThroughputBenchmark.runBenchmark(conf, new String[] {-op, all});
}
svn commit: r612191 - in /lucene/hadoop/trunk: CHANGES.txt src/java/org/apache/hadoop/mapred/TaskTracker.java
Author: shv Date: Tue Jan 15 11:00:57 2008 New Revision: 612191 URL: http://svn.apache.org/viewvc?rev=612191view=rev Log: HADOOP-2605. Remove bogus slash in task-tracker report bindAddress. Contributed by Konstantin Shvachko Modified: lucene/hadoop/trunk/CHANGES.txt lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TaskTracker.java Modified: lucene/hadoop/trunk/CHANGES.txt URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/CHANGES.txt?rev=612191r1=612190r2=612191view=diff == --- lucene/hadoop/trunk/CHANGES.txt (original) +++ lucene/hadoop/trunk/CHANGES.txt Tue Jan 15 11:00:57 2008 @@ -454,6 +454,9 @@ HADOOP-2494. Set +x on contrib/*/bin/* in packaged tar bundle. (stack via tomwhite) +HADOOP-2605. Remove bogus leading slash in task-tracker report bindAddress. +(Konstantin Shvachko) + Release 0.15.3 - (unreleased changes) BUG FIXES Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TaskTracker.java URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TaskTracker.java?rev=612191r1=612190r2=612191view=diff == --- lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TaskTracker.java (original) +++ lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TaskTracker.java Tue Jan 15 11:00:57 2008 @@ -431,7 +431,7 @@ // get the assigned address this.taskReportAddress = taskReportServer.getListenerAddress(); this.fConf.set(mapred.task.tracker.report.bindAddress, - taskReportAddress.toString()); +taskReportAddress.getHostName() + : + taskReportAddress.getPort()); LOG.info(TaskTracker up at: + this.taskReportAddress); this.taskTrackerName = tracker_ + localHostname + : + taskReportAddress;
svn commit: r612670 - in /lucene/hadoop/trunk: CHANGES.txt src/test/org/apache/hadoop/dfs/NNBenchWithoutMR.java
Author: shv
Date: Wed Jan 16 17:11:51 2008
New Revision: 612670
URL: http://svn.apache.org/viewvc?rev=612670view=rev
Log:
HADOOP-2449. A return of the non-MR version of NNBench. Contributed by Sanjay
Radia.
Added:
lucene/hadoop/trunk/src/test/org/apache/hadoop/dfs/NNBenchWithoutMR.java
Modified:
lucene/hadoop/trunk/CHANGES.txt
Modified: lucene/hadoop/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/lucene/hadoop/trunk/CHANGES.txt?rev=612670r1=612669r2=612670view=diff
==
--- lucene/hadoop/trunk/CHANGES.txt (original)
+++ lucene/hadoop/trunk/CHANGES.txt Wed Jan 16 17:11:51 2008
@@ -245,6 +245,9 @@
Add support for accessing instrumentation statistics via JMX.
(Sanjay radia via dhruba)
+HADOOP-2449. A return of the non-MR version of NNBench.
+(Sanjay Radia via shv)
+
OPTIMIZATIONS
HADOOP-1898. Release the lock protecting the last time of the last stack
Added: lucene/hadoop/trunk/src/test/org/apache/hadoop/dfs/NNBenchWithoutMR.java
URL:
http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/test/org/apache/hadoop/dfs/NNBenchWithoutMR.java?rev=612670view=auto
==
--- lucene/hadoop/trunk/src/test/org/apache/hadoop/dfs/NNBenchWithoutMR.java
(added)
+++ lucene/hadoop/trunk/src/test/org/apache/hadoop/dfs/NNBenchWithoutMR.java
Wed Jan 16 17:11:51 2008
@@ -0,0 +1,344 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * License); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an AS IS BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.dfs;
+
+import java.io.IOException;
+import java.util.Date;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.util.StringUtils;
+
+/**
+ * This program executes a specified operation that applies load to
+ * the NameNode. Possible operations include create/writing files,
+ * opening/reading files, renaming files, and deleting files.
+ *
+ * When run simultaneously on multiple nodes, this program functions
+ * as a stress-test and benchmark for namenode, especially when
+ * the number of bytes written to each file is small.
+ *
+ * This version does not use the map reduce framework
+ *
+ */
+public class NNBenchWithoutMR {
+
+ private static final Log LOG = LogFactory.getLog(
+org.apache.hadoop.dfs.NNBench);
+
+ // variable initialzed from command line arguments
+ private static long startTime = 0;
+ private static int numFiles = 0;
+ private static long bytesPerBlock = 1;
+ private static long blocksPerFile = 0;
+ private static long bytesPerFile = 1;
+ private static Path baseDir = null;
+
+ // variables initialized in main()
+ private static FileSystem fileSys = null;
+ private static Path taskDir = null;
+ private static String uniqueId = null;
+ private static byte[] buffer;
+ private static long maxExceptionsPerFile = 200;
+
+ /**
+ * Returns when the current number of seconds from the epoch equals
+ * the command line argument given by code-startTime/code.
+ * This allows multiple instances of this program, running on clock
+ * synchronized nodes, to start at roughly the same time.
+ */
+ static void barrier() {
+long sleepTime;
+while ((sleepTime = startTime - System.currentTimeMillis()) 0) {
+ try {
+Thread.sleep(sleepTime);
+ } catch (InterruptedException ex) {
+ }
+}
+ }
+
+ static private void handleException(String operation, Throwable e,
+ int singleFileExceptions) {
+LOG.warn(Exception while + operation + : +
+ StringUtils.stringifyException(e));
+if (singleFileExceptions = maxExceptionsPerFile) {
+ throw new RuntimeException(singleFileExceptions +
+ exceptions for a single file exceeds threshold. Aborting);
+}
+ }
+
+ /**
+ * Create and write to a given number of files. Repeat
svn commit: r612674 - in /lucene/hadoop/trunk: CHANGES.txt bin/hadoop
Author: shv Date: Wed Jan 16 17:22:55 2008 New Revision: 612674 URL: http://svn.apache.org/viewvc?rev=612674view=rev Log: HADOOP-1989. Remove 'datanodecluster' command from bin/hadoop. Contributed by Sanjay Radia. Modified: lucene/hadoop/trunk/CHANGES.txt lucene/hadoop/trunk/bin/hadoop Modified: lucene/hadoop/trunk/CHANGES.txt URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/CHANGES.txt?rev=612674r1=612673r2=612674view=diff == --- lucene/hadoop/trunk/CHANGES.txt (original) +++ lucene/hadoop/trunk/CHANGES.txt Wed Jan 16 17:22:55 2008 @@ -248,6 +248,9 @@ HADOOP-2449. A return of the non-MR version of NNBench. (Sanjay Radia via shv) +HADOOP-1989. Remove 'datanodecluster' command from bin/hadoop. +(Sanjay Radia via shv) + OPTIMIZATIONS HADOOP-1898. Release the lock protecting the last time of the last stack Modified: lucene/hadoop/trunk/bin/hadoop URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/bin/hadoop?rev=612674r1=612673r2=612674view=diff == --- lucene/hadoop/trunk/bin/hadoop (original) +++ lucene/hadoop/trunk/bin/hadoop Wed Jan 16 17:22:55 2008 @@ -36,7 +36,6 @@ echo secondarynamenoderun the DFS secondary namenode echo namenode run the DFS namenode echo datanode run a DFS datanode - echo datanodecluster run a DFS datanode cluster echo dfsadmin run a DFS admin client echo fsck run a DFS filesystem checking utility echo fs run a generic filesystem user client
svn commit: r612699 - in /lucene/hadoop/trunk: CHANGES.txt src/java/org/apache/hadoop/dfs/ClientProtocol.java src/java/org/apache/hadoop/dfs/DFSClient.java src/java/org/apache/hadoop/dfs/FSNamesystem.
Author: shv Date: Wed Jan 16 19:45:28 2008 New Revision: 612699 URL: http://svn.apache.org/viewvc?rev=612699view=rev Log: HADOOP-1742. Improve JavaDoc documentation for HDFS classes. Contributed by Konstantin Shvachko. Modified: lucene/hadoop/trunk/CHANGES.txt lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/ClientProtocol.java lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/DFSClient.java lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/FSNamesystem.java Modified: lucene/hadoop/trunk/CHANGES.txt URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/CHANGES.txt?rev=612699r1=612698r2=612699view=diff == --- lucene/hadoop/trunk/CHANGES.txt (original) +++ lucene/hadoop/trunk/CHANGES.txt Wed Jan 16 19:45:28 2008 @@ -251,6 +251,9 @@ HADOOP-1989. Remove 'datanodecluster' command from bin/hadoop. (Sanjay Radia via shv) +HADOOP-1742. Improve JavaDoc documentation for ClientProtocol, DFSClient, +and FSNamesystem. (Konstantin Shvachko) + OPTIMIZATIONS HADOOP-1898. Release the lock protecting the last time of the last stack Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/ClientProtocol.java URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/ClientProtocol.java?rev=612699r1=612698r2=612699view=diff == --- lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/ClientProtocol.java (original) +++ lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/ClientProtocol.java Wed Jan 16 19:45:28 2008 @@ -23,7 +23,8 @@ import org.apache.hadoop.fs.permission.*; /** - * ClientProtocol is used by a piece of DFS user code to communicate + * ClientProtocol is used by user code via + * [EMAIL PROTECTED] DistributedFileSystem} class to communicate * with the NameNode. User code can manipulate the directory namespace, * as well as open/close file streams, etc. * @@ -32,13 +33,8 @@ /** * Compared to the previous version the following changes have been introduced: - * 16 : removed deprecated obtainLock() and releaseLock(). - * 17 : getBlockSize replaced by getPreferredBlockSize - * 18 : datanodereport returns dead, live or all nodes. - * 19 : rollEditLog() returns a token to uniquely identify the editfile. - * 20 : getContentLength returns the total size in bytes of a directory subtree - * 21 : add lease holder as a parameter in abandonBlock(...) - * 22 : Serialization of FileStatus has changed. + * (Only the latest change is reflected. + * The log of historical changes can be retrieved from the svn). * 23 : added setOwner(...) and setPermission(...); changed create(...) and mkdir(...) */ public static final long versionID = 23L; @@ -47,12 +43,14 @@ // File contents /// /** - * Open an existing file and get block locations within the specified range. + * Open an existing file for read and get block locations within + * the specified range. + * p * Return [EMAIL PROTECTED] LocatedBlocks} which contains * file length, blocks and their locations. * DataNode locations for each block are sorted by * the distance to the client's address. - * + * p * The client will then have to contact * one of the indicated DataNodes to obtain the actual data. There * is no need to call close() or any other function after @@ -86,24 +84,33 @@ long length) throws IOException; /** - * Create a new file. Get back block and datanode info, - * which describes where the first block should be written. - * - * Successfully calling this method prevents any other - * client from creating a file under the given name, but - * the caller must invoke complete() for the file to be - * added to the filesystem. - * + * Create a new file entry in the namespace. + * p + * This will create an empty file specified by the source path. + * The path should reflect a full path originated at the root. + * The name-node does not have a notion of current directory for a client. + * p + * Once created, the file is visible and available for read to other clients. + * Although, other clients cannot [EMAIL PROTECTED] #delete(String)}, re-create or + * [EMAIL PROTECTED] #rename(String, String)} it until the file is completed or + * abandoned implicitely by [EMAIL PROTECTED] #abandonFileInProgress(String, String)} + * or explicitely as a result of lease expiration. + * p * Blocks have a maximum size. Clients that intend to - * create multi-block files must also use reportWrittenBlock() - * and addBlock(). - * - * If permission denied, - * an [EMAIL PROTECTED] AccessControlException} will be thrown as an - * [EMAIL PROTECTED
