Author: omalley
Date: Tue Mar 8 04:41:19 2011
New Revision: 1079166
URL: http://svn.apache.org/viewvc?rev=1079166&view=rev
Log:
commit 90bd40ff0cc5a45b82759e227c46599bb4df6c22
Author: Jitendra Nath Pandey <jitendra@sufferhome-lm.(none)>
Date: Tue Feb 1 16:44:50 2011 -0800
HADOOP-6432. A few files were missed in the previous commit.
Add Statistics support in FileContext.
+++ b/YAHOO-CHANGES.txt
+ HADOOP-6432. Add Statistics support in FileContext. (jitendra)
+
Modified:
hadoop/common/branches/yahoo-merge/src/java/org/apache/hadoop/fs/AbstractFileSystem.java
hadoop/common/branches/yahoo-merge/src/java/org/apache/hadoop/fs/FileContext.java
hadoop/common/branches/yahoo-merge/src/java/org/apache/hadoop/fs/FileSystem.java
Modified:
hadoop/common/branches/yahoo-merge/src/java/org/apache/hadoop/fs/AbstractFileSystem.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/yahoo-merge/src/java/org/apache/hadoop/fs/AbstractFileSystem.java?rev=1079166&r1=1079165&r2=1079166&view=diff
==============================================================================
---
hadoop/common/branches/yahoo-merge/src/java/org/apache/hadoop/fs/AbstractFileSystem.java
(original)
+++
hadoop/common/branches/yahoo-merge/src/java/org/apache/hadoop/fs/AbstractFileSystem.java
Tue Mar 8 04:41:19 2011
@@ -24,7 +24,7 @@ import java.lang.reflect.Constructor;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.EnumSet;
-import java.util.IdentityHashMap;
+import java.util.HashMap;
import java.util.Map;
import java.util.NoSuchElementException;
import java.util.StringTokenizer;
@@ -60,9 +60,8 @@ public abstract class AbstractFileSystem
static final Log LOG = LogFactory.getLog(AbstractFileSystem.class);
/** Recording statistics per a file system class. */
- private static final Map<Class<? extends AbstractFileSystem>, Statistics>
- STATISTICS_TABLE =
- new IdentityHashMap<Class<? extends AbstractFileSystem>, Statistics>();
+ private static final Map<URI, Statistics>
+ STATISTICS_TABLE = new HashMap<URI, Statistics>();
/** Cache of constructors for each file system class. */
private static final Map<Class<?>, Constructor<?>> CONSTRUCTOR_CACHE =
@@ -144,35 +143,68 @@ public abstract class AbstractFileSystem
}
return (AbstractFileSystem) newInstance(clazz, uri, conf);
}
-
-
+
/**
* Get the statistics for a particular file system.
- * @param cls the class to lookup
+ *
+ * @param uri
+ * used as key to lookup STATISTICS_TABLE. Only scheme and authority
+ * part of the uri are used.
* @return a statistics object
*/
- public static synchronized Statistics getStatistics(String scheme,
- Class<? extends AbstractFileSystem> cls) {
- Statistics result = STATISTICS_TABLE.get(cls);
+ protected static synchronized Statistics getStatistics(URI uri) {
+ String scheme = uri.getScheme();
+ if (scheme == null) {
+ throw new IllegalArgumentException("Scheme not defined in the uri: "
+ + uri);
+ }
+ URI baseUri = getBaseUri(uri);
+ Statistics result = STATISTICS_TABLE.get(baseUri);
if (result == null) {
result = new Statistics(scheme);
- STATISTICS_TABLE.put(cls, result);
+ STATISTICS_TABLE.put(baseUri, result);
}
return result;
}
+ private static URI getBaseUri(URI uri) {
+ String scheme = uri.getScheme();
+ String authority = uri.getAuthority();
+ String baseUriString = scheme + "://";
+ if (authority != null) {
+ baseUriString = baseUriString + authority;
+ } else {
+ baseUriString = baseUriString + "/";
+ }
+ return URI.create(baseUriString);
+ }
+
public static synchronized void clearStatistics() {
for(Statistics stat: STATISTICS_TABLE.values()) {
stat.reset();
}
}
+ /**
+ * Prints statistics for all file systems.
+ */
public static synchronized void printStatistics() {
- for (Map.Entry<Class<? extends AbstractFileSystem>, Statistics> pair:
- STATISTICS_TABLE.entrySet()) {
- System.out.println(" FileSystem " + pair.getKey().getName() +
- ": " + pair.getValue());
+ for (Map.Entry<URI, Statistics> pair : STATISTICS_TABLE.entrySet()) {
+ System.out.println(" FileSystem " + pair.getKey().getScheme() + "://"
+ + pair.getKey().getAuthority() + ": " + pair.getValue());
+ }
+ }
+
+ protected static synchronized Map<URI, Statistics> getAllStatistics() {
+ Map<URI, Statistics> statsMap = new HashMap<URI, Statistics>(
+ STATISTICS_TABLE.size());
+ for (Map.Entry<URI, Statistics> pair : STATISTICS_TABLE.entrySet()) {
+ URI key = pair.getKey();
+ Statistics value = pair.getValue();
+ Statistics newStatsObj = new Statistics(value);
+ statsMap.put(URI.create(key.toString()), newStatsObj);
}
+ return statsMap;
}
/**
@@ -211,7 +243,7 @@ public abstract class AbstractFileSystem
final boolean authorityNeeded, final int defaultPort)
throws URISyntaxException {
myUri = getUri(uri, supportedScheme, authorityNeeded, defaultPort);
- statistics = getStatistics(supportedScheme, getClass());
+ statistics = getStatistics(uri);
}
/**
Modified:
hadoop/common/branches/yahoo-merge/src/java/org/apache/hadoop/fs/FileContext.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/yahoo-merge/src/java/org/apache/hadoop/fs/FileContext.java?rev=1079166&r1=1079165&r2=1079166&view=diff
==============================================================================
---
hadoop/common/branches/yahoo-merge/src/java/org/apache/hadoop/fs/FileContext.java
(original)
+++
hadoop/common/branches/yahoo-merge/src/java/org/apache/hadoop/fs/FileContext.java
Tue Mar 8 04:41:19 2011
@@ -40,6 +40,7 @@ import org.apache.hadoop.HadoopIllegalAr
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem.Statistics;
import org.apache.hadoop.fs.Options.CreateOpts;
import org.apache.hadoop.fs.Options.Rename;
import org.apache.hadoop.fs.permission.FsPermission;
@@ -812,8 +813,8 @@ public final class FileContext {
*
* @throws AccessControlException If access is denied
* @throws FileAlreadyExistsException If <code>dst</code> already exists and
- * <code>options</options> has {@link Rename#OVERWRITE} option
- * false.
+ * <code>options</options> has {@link Options.Rename#OVERWRITE}
+ * option false.
* @throws FileNotFoundException If <code>src</code> does not exist
* @throws ParentNotDirectoryException If parent of <code>dst</code> is not a
* directory
@@ -2252,4 +2253,40 @@ public final class FileContext {
return in;
}
}
+
+ /**
+ * Get the statistics for a particular file system
+ *
+ * @param uri
+ * the uri to lookup the statistics. Only scheme and authority part
+ * of the uri are used as the key to store and lookup.
+ * @return a statistics object
+ */
+ public static Statistics getStatistics(URI uri) {
+ return AbstractFileSystem.getStatistics(uri);
+ }
+
+ /**
+ * Clears all the statistics stored in AbstractFileSystem, for all the file
+ * systems.
+ */
+ public static void clearStatistics() {
+ AbstractFileSystem.clearStatistics();
+ }
+
+ /**
+ * Prints the statistics to standard output. File System is identified by the
+ * scheme and authority.
+ */
+ public static void printStatistics() {
+ AbstractFileSystem.printStatistics();
+ }
+
+ /**
+ * @return Map of uri and statistics for each filesystem instantiated. The
uri
+ * consists of scheme and authority for the filesystem.
+ */
+ public static Map<URI, Statistics> getAllStatistics() {
+ return AbstractFileSystem.getAllStatistics();
+ }
}
Modified:
hadoop/common/branches/yahoo-merge/src/java/org/apache/hadoop/fs/FileSystem.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/yahoo-merge/src/java/org/apache/hadoop/fs/FileSystem.java?rev=1079166&r1=1079165&r2=1079166&view=diff
==============================================================================
---
hadoop/common/branches/yahoo-merge/src/java/org/apache/hadoop/fs/FileSystem.java
(original)
+++
hadoop/common/branches/yahoo-merge/src/java/org/apache/hadoop/fs/FileSystem.java
Tue Mar 8 04:41:19 2011
@@ -2008,6 +2008,18 @@ public abstract class FileSystem extends
}
/**
+ * Copy constructor.
+ *
+ * @param st
+ * The input Statistics object which is cloned.
+ */
+ public Statistics(Statistics st) {
+ this.scheme = st.scheme;
+ this.bytesRead = new AtomicLong(st.bytesRead.longValue());
+ this.bytesWritten = new AtomicLong(st.bytesWritten.longValue());
+ }
+
+ /**
* Increment the bytes read in the statistics
* @param newBytes the additional bytes read
*/
