Author: todd
Date: Mon Jan 14 21:12:16 2013
New Revision: 1433133
URL: http://svn.apache.org/viewvc?rev=1433133&view=rev
Log:
Merge trunk into branch.
Fixed a couple trivial conflicts due to HDFS-4363, which moved some methods
from HdfsProtoUtil to PBHelper
Added:
hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/IdGenerator.java
- copied unchanged from r1433128,
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/IdGenerator.java
hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCompressionStreamReuse.java
- copied unchanged from r1433128,
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCompressionStreamReuse.java
Modified:
hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/CHANGES.txt
(contents, props changed)
hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/pom.xml
hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/config.h.cmake
hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/main/docs/
(props changed)
hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/main/java/
(props changed)
hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileStatus.java
hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FTPFileSystem.java
hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/local/RawLocalFs.java
hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/FsPermission.java
hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ActiveStandbyElector.java
hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/GzipCodec.java
hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/SequentialNumber.java
hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/main/proto/Security.proto
hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/main/resources/META-INF/services/org.apache.hadoop.security.SecurityInfo
hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/test/core/
(props changed)
hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextPermissionBase.java
hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestDelegationTokenRenewer.java
hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileStatus.java
hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFSFileContextMainOperations.java
hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystemPermission.java
hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/test/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier
hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/test/resources/kdc/killKdc.sh
hadoop/common/branches/HDFS-347/hadoop-common-project/pom.xml
Modified:
hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1433133&r1=1433132&r2=1433133&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/CHANGES.txt
(original)
+++
hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/CHANGES.txt
Mon Jan 14 21:12:16 2013
@@ -148,6 +148,8 @@ Trunk (Unreleased)
BUG FIXES
+ HADOOP-8419. Fixed GzipCode NPE reset for IBM JDK. (Yu Li via eyang)
+
HADOOP-9041. FsUrlStreamHandlerFactory could cause an infinite loop in
FileSystem initialization. (Yanbo Liang and Radim Kolar via llu)
@@ -306,9 +308,14 @@ Trunk (Unreleased)
HADOOP-9131. Turn off TestLocalFileSystem#testListStatusWithColons on
Windows. (Chris Nauroth via suresh)
- HADOOP-8957 AbstractFileSystem#IsValidName should be overridden for
+ HADOOP-8957. AbstractFileSystem#IsValidName should be overridden for
embedded file systems like ViewFs (Chris Nauroth via Sanjay Radia)
+ HADOOP-9139. improve killKdc.sh (Ivan A. Veselovsky via bobby)
+
+ HADOOP-9202. test-patch.sh fails during mvn eclipse:eclipse if patch adds
+ a new module to the build (Chris Nauroth via bobby)
+
OPTIMIZATIONS
HADOOP-7761. Improve the performance of raw comparisons. (todd)
@@ -426,6 +433,9 @@ Release 2.0.3-alpha - Unreleased
HADOOP-9119. Add test to FileSystemContractBaseTest to verify integrity
of overwritten files. (Steve Loughran via suresh)
+ HADOOP-9192. Move token related request/response messages to common.
+ (suresh)
+
OPTIMIZATIONS
HADOOP-8866. SampleQuantiles#query is O(N^2) instead of O(N). (Andrew Wang
@@ -525,6 +535,11 @@ Release 2.0.3-alpha - Unreleased
HADOOP-9181. Set daemon flag for HttpServer's QueuedThreadPool.
(Liang Xie via suresh)
+ HADOOP-9155. FsPermission should have different default value, 777 for
+ directory and 666 for file. (Binglin Chang via atm)
+
+ HADOOP-9183. Potential deadlock in ActiveStandbyElector. (tomwhite)
+
Release 2.0.2-alpha - 2012-09-07
INCOMPATIBLE CHANGES
@@ -1238,6 +1253,8 @@ Release 0.23.6 - UNRELEASED
HADOOP-9105. FsShell -moveFromLocal erroneously fails (daryn via bobby)
+ HADOOP-9097. Maven RAT plugin is not checking all source files (tgraves)
+
Release 0.23.5 - UNRELEASED
INCOMPATIBLE CHANGES
Propchange:
hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/CHANGES.txt
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt:r1430995-1433128
Modified:
hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/pom.xml
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/pom.xml?rev=1433133&r1=1433132&r2=1433133&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/pom.xml
(original)
+++ hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/pom.xml
Mon Jan 14 21:12:16 2013
@@ -445,10 +445,11 @@
<exclude>dev-support/jdiff/**</exclude>
<exclude>src/main/native/*</exclude>
<exclude>src/main/native/config/*</exclude>
-
<exclude>src/main/resources/META-INF/services/org.apache.hadoop.security.SecurityInfo</exclude>
<exclude>src/main/native/m4/*</exclude>
<exclude>src/test/empty-file</exclude>
<exclude>src/test/all-tests</exclude>
+ <exclude>src/test/resources/kdc/ldif/users.ldif</exclude>
+
<exclude>src/main/native/src/org/apache/hadoop/io/compress/lz4/lz4.c</exclude>
</excludes>
</configuration>
</plugin>
Modified:
hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/config.h.cmake
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/config.h.cmake?rev=1433133&r1=1433132&r2=1433133&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/config.h.cmake
(original)
+++
hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/config.h.cmake
Mon Jan 14 21:12:16 2013
@@ -1,3 +1,20 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
#ifndef CONFIG_H
#define CONFIG_H
Propchange:
hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/main/docs/
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/docs:r1430995-1433128
Propchange:
hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/main/java/
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java:r1430995-1433128
Modified:
hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java?rev=1433133&r1=1433132&r2=1433133&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
(original)
+++
hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
Mon Jan 14 21:12:16 2013
@@ -172,7 +172,25 @@ import org.apache.hadoop.util.ShutdownHo
public final class FileContext {
public static final Log LOG = LogFactory.getLog(FileContext.class);
+ /**
+ * Default permission for directory and symlink
+ * In previous versions, this default permission was also used to
+ * create files, so files created end up with ugo+x permission.
+ * See HADOOP-9155 for detail.
+ * Two new constants are added to solve this, please use
+ * {@link FileContext#DIR_DEFAULT_PERM} for directory, and use
+ * {@link FileContext#FILE_DEFAULT_PERM} for file.
+ * This constant is kept for compatibility.
+ */
public static final FsPermission DEFAULT_PERM = FsPermission.getDefault();
+ /**
+ * Default permission for directory
+ */
+ public static final FsPermission DIR_DEFAULT_PERM =
FsPermission.getDirDefault();
+ /**
+ * Default permission for file
+ */
+ public static final FsPermission FILE_DEFAULT_PERM =
FsPermission.getFileDefault();
/**
* Priority of the FileContext shutdown hook.
@@ -656,7 +674,7 @@ public final class FileContext {
CreateOpts.Perms permOpt =
(CreateOpts.Perms) CreateOpts.getOpt(CreateOpts.Perms.class, opts);
FsPermission permission = (permOpt != null) ? permOpt.getValue() :
- FsPermission.getDefault();
+ FILE_DEFAULT_PERM;
permission = permission.applyUMask(umask);
final CreateOpts[] updatedOpts =
@@ -704,7 +722,7 @@ public final class FileContext {
IOException {
final Path absDir = fixRelativePart(dir);
final FsPermission absFerms = (permission == null ?
- FsPermission.getDefault() : permission).applyUMask(umask);
+ FsPermission.getDirDefault() : permission).applyUMask(umask);
new FSLinkResolver<Void>() {
@Override
public Void next(final AbstractFileSystem fs, final Path p)
@@ -2157,7 +2175,7 @@ public final class FileContext {
FileStatus fs = FileContext.this.getFileStatus(qSrc);
if (fs.isDirectory()) {
checkDependencies(qSrc, qDst);
- mkdir(qDst, FsPermission.getDefault(), true);
+ mkdir(qDst, FsPermission.getDirDefault(), true);
FileStatus[] contents = listStatus(qSrc);
for (FileStatus content : contents) {
copy(makeQualified(content.getPath()), makeQualified(new Path(qDst,
Modified:
hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileStatus.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileStatus.java?rev=1433133&r1=1433132&r2=1433133&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileStatus.java
(original)
+++
hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileStatus.java
Mon Jan 14 21:12:16 2013
@@ -79,8 +79,15 @@ public class FileStatus implements Writa
this.blocksize = blocksize;
this.modification_time = modification_time;
this.access_time = access_time;
- this.permission = (permission == null) ?
- FsPermission.getDefault() : permission;
+ if (permission != null) {
+ this.permission = permission;
+ } else if (isdir) {
+ this.permission = FsPermission.getDirDefault();
+ } else if (symlink!=null) {
+ this.permission = FsPermission.getDefault();
+ } else {
+ this.permission = FsPermission.getFileDefault();
+ }
this.owner = (owner == null) ? "" : owner;
this.group = (group == null) ? "" : group;
this.symlink = symlink;
@@ -217,7 +224,7 @@ public class FileStatus implements Writa
*/
protected void setPermission(FsPermission permission) {
this.permission = (permission == null) ?
- FsPermission.getDefault() : permission;
+ FsPermission.getFileDefault() : permission;
}
/**
Modified:
hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java?rev=1433133&r1=1433132&r2=1433133&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
(original)
+++
hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
Mon Jan 14 21:12:16 2013
@@ -850,7 +850,7 @@ public abstract class FileSystem extends
long blockSize,
Progressable progress
) throws IOException {
- return this.create(f, FsPermission.getDefault().applyUMask(
+ return this.create(f, FsPermission.getFileDefault().applyUMask(
FsPermission.getUMask(getConf())), overwrite, bufferSize,
replication, blockSize, progress);
}
@@ -1030,7 +1030,7 @@ public abstract class FileSystem extends
boolean overwrite,
int bufferSize, short replication, long blockSize,
Progressable progress) throws IOException {
- return this.createNonRecursive(f, FsPermission.getDefault(),
+ return this.createNonRecursive(f, FsPermission.getFileDefault(),
overwrite, bufferSize, replication, blockSize, progress);
}
@@ -1866,7 +1866,7 @@ public abstract class FileSystem extends
* Call {@link #mkdirs(Path, FsPermission)} with default permission.
*/
public boolean mkdirs(Path f) throws IOException {
- return mkdirs(f, FsPermission.getDefault());
+ return mkdirs(f, FsPermission.getDirDefault());
}
/**
Modified:
hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FTPFileSystem.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FTPFileSystem.java?rev=1433133&r1=1433132&r2=1433133&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FTPFileSystem.java
(original)
+++
hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FTPFileSystem.java
Mon Jan 14 21:12:16 2013
@@ -224,7 +224,7 @@ public class FTPFileSystem extends FileS
}
Path parent = absolute.getParent();
- if (parent == null || !mkdirs(client, parent, FsPermission.getDefault())) {
+ if (parent == null || !mkdirs(client, parent,
FsPermission.getDirDefault())) {
parent = (parent == null) ? new Path("/") : parent;
disconnect(client);
throw new IOException("create(): Mkdirs failed to create: " + parent);
@@ -484,7 +484,7 @@ public class FTPFileSystem extends FileS
if (!exists(client, absolute)) {
Path parent = absolute.getParent();
created = (parent == null || mkdirs(client, parent, FsPermission
- .getDefault()));
+ .getDirDefault()));
if (created) {
String parentDir = parent.toUri().getPath();
client.changeWorkingDirectory(parentDir);
Modified:
hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/local/RawLocalFs.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/local/RawLocalFs.java?rev=1433133&r1=1433132&r2=1433133&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/local/RawLocalFs.java
(original)
+++
hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/local/RawLocalFs.java
Mon Jan 14 21:12:16 2013
@@ -85,7 +85,7 @@ public class RawLocalFs extends Delegate
"system: "+target.toString());
}
if (createParent) {
- mkdir(link.getParent(), FsPermission.getDefault(), true);
+ mkdir(link.getParent(), FsPermission.getDirDefault(), true);
}
// NB: Use createSymbolicLink in java.nio.file.Path once available
try {
Modified:
hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/FsPermission.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/FsPermission.java?rev=1433133&r1=1433132&r2=1433133&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/FsPermission.java
(original)
+++
hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/FsPermission.java
Mon Jan 14 21:12:16 2013
@@ -275,12 +275,35 @@ public class FsPermission implements Wri
conf.setInt(DEPRECATED_UMASK_LABEL, umask.toShort());
}
- /** Get the default permission. */
+ /**
+ * Get the default permission for directory and symlink.
+ * In previous versions, this default permission was also used to
+ * create files, so files created end up with ugo+x permission.
+ * See HADOOP-9155 for detail.
+ * Two new methods are added to solve this, please use
+ * {@link FsPermission#getDirDefault()} for directory, and use
+ * {@link FsPermission#getFileDefault()} for file.
+ * This method is kept for compatibility.
+ */
public static FsPermission getDefault() {
return new FsPermission((short)00777);
}
/**
+ * Get the default permission for directory.
+ */
+ public static FsPermission getDirDefault() {
+ return new FsPermission((short)00777);
+ }
+
+ /**
+ * Get the default permission for file.
+ */
+ public static FsPermission getFileDefault() {
+ return new FsPermission((short)00666);
+ }
+
+ /**
* Create a FsPermission from a Unix symbolic permission string
* @param unixSymbolicPermission e.g. "-rw-rw-rw-"
*/
Modified:
hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ActiveStandbyElector.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ActiveStandbyElector.java?rev=1433133&r1=1433132&r2=1433133&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ActiveStandbyElector.java
(original)
+++
hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ActiveStandbyElector.java
Mon Jan 14 21:12:16 2013
@@ -613,7 +613,7 @@ public class ActiveStandbyElector implem
// Unfortunately, the ZooKeeper constructor connects to ZooKeeper and
// may trigger the Connected event immediately. So, if we register the
// watcher after constructing ZooKeeper, we may miss that event. Instead,
- // we construct the watcher first, and have it queue any events it receives
+ // we construct the watcher first, and have it block any events it receives
// before we can set its ZooKeeper reference.
WatcherWithClientRef watcher = new WatcherWithClientRef();
ZooKeeper zk = new ZooKeeper(zkHostPort, zkSessionTimeout, watcher);
@@ -1002,19 +1002,17 @@ public class ActiveStandbyElector implem
private CountDownLatch hasReceivedEvent = new CountDownLatch(1);
/**
- * If any events arrive before the reference to ZooKeeper is set,
- * they get queued up and later forwarded when the reference is
- * available.
+ * Latch used to wait until the reference to ZooKeeper is set.
*/
- private final List<WatchedEvent> queuedEvents = Lists.newLinkedList();
+ private CountDownLatch hasSetZooKeeper = new CountDownLatch(1);
private WatcherWithClientRef() {
}
private WatcherWithClientRef(ZooKeeper zk) {
- this.zk = zk;
+ setZooKeeperRef(zk);
}
-
+
/**
* Waits for the next event from ZooKeeper to arrive.
*
@@ -1029,9 +1027,7 @@ public class ActiveStandbyElector implem
if (!hasReceivedEvent.await(connectionTimeoutMs,
TimeUnit.MILLISECONDS)) {
LOG.error("Connection timed out: couldn't connect to ZooKeeper in "
+ connectionTimeoutMs + " milliseconds");
- synchronized (this) {
- zk.close();
- }
+ zk.close();
throw KeeperException.create(Code.CONNECTIONLOSS);
}
} catch (InterruptedException e) {
@@ -1041,29 +1037,18 @@ public class ActiveStandbyElector implem
}
}
- private synchronized void setZooKeeperRef(ZooKeeper zk) {
+ private void setZooKeeperRef(ZooKeeper zk) {
Preconditions.checkState(this.zk == null,
"zk already set -- must be set exactly once");
this.zk = zk;
-
- for (WatchedEvent e : queuedEvents) {
- forwardEvent(e);
- }
- queuedEvents.clear();
+ hasSetZooKeeper.countDown();
}
@Override
- public synchronized void process(WatchedEvent event) {
- if (zk != null) {
- forwardEvent(event);
- } else {
- queuedEvents.add(event);
- }
- }
-
- private void forwardEvent(WatchedEvent event) {
+ public void process(WatchedEvent event) {
hasReceivedEvent.countDown();
try {
+ hasSetZooKeeper.await(zkSessionTimeout, TimeUnit.MILLISECONDS);
ActiveStandbyElector.this.processWatchEvent(
zk, event);
} catch (Throwable t) {
Modified:
hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/GzipCodec.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/GzipCodec.java?rev=1433133&r1=1433132&r2=1433133&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/GzipCodec.java
(original)
+++
hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/GzipCodec.java
Mon Jan 14 21:12:16 2013
@@ -40,14 +40,74 @@ public class GzipCodec extends DefaultCo
protected static class GzipOutputStream extends CompressorStream {
private static class ResetableGZIPOutputStream extends GZIPOutputStream {
-
+ private static final int TRAILER_SIZE = 8;
+ public static final String JVMVendor= System.getProperty("java.vendor");
+ public static final String JVMVersion=
System.getProperty("java.version");
+ private static final boolean HAS_BROKEN_FINISH =
+ (JVMVendor.contains("IBM") && JVMVersion.contains("1.6.0"));
+
public ResetableGZIPOutputStream(OutputStream out) throws IOException {
super(out);
}
-
+
public void resetState() throws IOException {
def.reset();
}
+
+ /**
+ * Override this method for HADOOP-8419.
+ * Override because IBM implementation calls def.end() which
+ * causes problem when reseting the stream for reuse.
+ *
+ */
+ @Override
+ public void finish() throws IOException {
+ if (HAS_BROKEN_FINISH) {
+ if (!def.finished()) {
+ def.finish();
+ while (!def.finished()) {
+ int i = def.deflate(this.buf, 0, this.buf.length);
+ if ((def.finished()) && (i <= this.buf.length - TRAILER_SIZE)) {
+ writeTrailer(this.buf, i);
+ i += TRAILER_SIZE;
+ out.write(this.buf, 0, i);
+
+ return;
+ }
+ if (i > 0) {
+ out.write(this.buf, 0, i);
+ }
+ }
+
+ byte[] arrayOfByte = new byte[TRAILER_SIZE];
+ writeTrailer(arrayOfByte, 0);
+ out.write(arrayOfByte);
+ }
+ } else {
+ super.finish();
+ }
+ }
+
+ /** re-implement for HADOOP-8419 because the relative method in jdk is
invisible */
+ private void writeTrailer(byte[] paramArrayOfByte, int paramInt)
+ throws IOException {
+ writeInt((int)this.crc.getValue(), paramArrayOfByte, paramInt);
+ writeInt(this.def.getTotalIn(), paramArrayOfByte, paramInt + 4);
+ }
+
+ /** re-implement for HADOOP-8419 because the relative method in jdk is
invisible */
+ private void writeInt(int paramInt1, byte[] paramArrayOfByte, int
paramInt2)
+ throws IOException {
+ writeShort(paramInt1 & 0xFFFF, paramArrayOfByte, paramInt2);
+ writeShort(paramInt1 >> 16 & 0xFFFF, paramArrayOfByte, paramInt2 + 2);
+ }
+
+ /** re-implement for HADOOP-8419 because the relative method in jdk is
invisible */
+ private void writeShort(int paramInt1, byte[] paramArrayOfByte, int
paramInt2)
+ throws IOException {
+ paramArrayOfByte[paramInt2] = (byte)(paramInt1 & 0xFF);
+ paramArrayOfByte[(paramInt2 + 1)] = (byte)(paramInt1 >> 8 & 0xFF);
+ }
}
public GzipOutputStream(OutputStream out) throws IOException {
Modified:
hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/SequentialNumber.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/SequentialNumber.java?rev=1433133&r1=1433132&r2=1433133&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/SequentialNumber.java
(original)
+++
hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/SequentialNumber.java
Mon Jan 14 21:12:16 2013
@@ -27,7 +27,7 @@ import org.apache.hadoop.classification.
* This class is thread safe.
*/
@InterfaceAudience.Private
-public abstract class SequentialNumber {
+public abstract class SequentialNumber implements IdGenerator {
private final AtomicLong currentValue;
/** Create a new instance with the given initial value. */
Modified:
hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/main/proto/Security.proto
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/main/proto/Security.proto?rev=1433133&r1=1433132&r2=1433133&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/main/proto/Security.proto
(original)
+++
hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/main/proto/Security.proto
Mon Jan 14 21:12:16 2013
@@ -32,3 +32,26 @@ message TokenProto {
required string service = 4;
}
+message GetDelegationTokenRequestProto {
+ required string renewer = 1;
+}
+
+message GetDelegationTokenResponseProto {
+ optional hadoop.common.TokenProto token = 1;
+}
+
+message RenewDelegationTokenRequestProto {
+ required hadoop.common.TokenProto token = 1;
+}
+
+message RenewDelegationTokenResponseProto {
+ required uint64 newExpiryTime = 1;
+}
+
+message CancelDelegationTokenRequestProto {
+ required hadoop.common.TokenProto token = 1;
+}
+
+message CancelDelegationTokenResponseProto { // void response
+}
+
Modified:
hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/main/resources/META-INF/services/org.apache.hadoop.security.SecurityInfo
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/main/resources/META-INF/services/org.apache.hadoop.security.SecurityInfo?rev=1433133&r1=1433132&r2=1433133&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/main/resources/META-INF/services/org.apache.hadoop.security.SecurityInfo
(original)
+++
hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/main/resources/META-INF/services/org.apache.hadoop.security.SecurityInfo
Mon Jan 14 21:12:16 2013
@@ -1 +1,14 @@
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
org.apache.hadoop.security.AnnotatedSecurityInfo
Propchange:
hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/test/core/
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/core:r1430995-1433128
Modified:
hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextPermissionBase.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextPermissionBase.java?rev=1433133&r1=1433132&r2=1433133&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextPermissionBase.java
(original)
+++
hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextPermissionBase.java
Mon Jan 14 21:12:16 2013
@@ -95,7 +95,7 @@ public abstract class FileContextPermiss
String filename = "foo";
Path f = getTestRootPath(fc, filename);
createFile(fc, filename);
- doFilePermissionCheck(FileContext.DEFAULT_PERM.applyUMask(fc.getUMask()),
+
doFilePermissionCheck(FileContext.FILE_DEFAULT_PERM.applyUMask(fc.getUMask()),
fc.getFileStatus(f).getPermission());
}
Modified:
hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestDelegationTokenRenewer.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestDelegationTokenRenewer.java?rev=1433133&r1=1433132&r2=1433133&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestDelegationTokenRenewer.java
(original)
+++
hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestDelegationTokenRenewer.java
Mon Jan 14 21:12:16 2013
@@ -1,3 +1,20 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
package org.apache.hadoop.fs;
import java.io.FileNotFoundException;
Modified:
hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileStatus.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileStatus.java?rev=1433133&r1=1433132&r2=1433133&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileStatus.java
(original)
+++
hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileStatus.java
Mon Jan 14 21:12:16 2013
@@ -121,7 +121,7 @@ public class TestFileStatus {
FileStatus fileStatus = new FileStatus(LENGTH, isdir,
REPLICATION, BLKSIZE, MTIME, PATH);
validateAccessors(fileStatus, LENGTH, isdir, REPLICATION, BLKSIZE, MTIME,
- 0, FsPermission.getDefault(), "", "", null, PATH);
+ 0, FsPermission.getDirDefault(), "", "", null, PATH);
}
/**
@@ -131,7 +131,7 @@ public class TestFileStatus {
public void constructorBlank() throws IOException {
FileStatus fileStatus = new FileStatus();
validateAccessors(fileStatus, 0, false, 0, 0, 0,
- 0, FsPermission.getDefault(), "", "", null, null);
+ 0, FsPermission.getFileDefault(), "", "", null, null);
}
/**
Modified:
hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFSFileContextMainOperations.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFSFileContextMainOperations.java?rev=1433133&r1=1433132&r2=1433133&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFSFileContextMainOperations.java
(original)
+++
hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFSFileContextMainOperations.java
Mon Jan 14 21:12:16 2013
@@ -24,6 +24,8 @@ import org.apache.hadoop.conf.Configurat
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
+import org.apache.hadoop.fs.FileContextTestHelper;
+import org.apache.hadoop.fs.permission.FsPermission;
public class TestLocalFSFileContextMainOperations extends
FileContextMainOperationsBaseTest {
@@ -47,4 +49,14 @@ public class TestLocalFSFileContextMainO
FileContext fc1 = FileContext.getLocalFSFileContext();
Assert.assertTrue(fc1 != fc);
}
+
+ @Test
+ public void testDefaultFilePermission() throws IOException {
+ Path file = FileContextTestHelper.getTestRootPath(fc,
+ "testDefaultFilePermission");
+ FileContextTestHelper.createFile(fc, file);
+ FsPermission expect =
FileContext.FILE_DEFAULT_PERM.applyUMask(fc.getUMask());
+ Assert.assertEquals(expect, fc.getFileStatus(file)
+ .getPermission());
+ }
}
Modified:
hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystemPermission.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystemPermission.java?rev=1433133&r1=1433132&r2=1433133&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystemPermission.java
(original)
+++
hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystemPermission.java
Mon Jan 14 21:12:16 2013
@@ -73,7 +73,7 @@ public class TestLocalFileSystemPermissi
try {
FsPermission initialPermission = getPermission(localfs, f);
System.out.println(filename + ": " + initialPermission);
-
assertEquals(FsPermission.getDefault().applyUMask(FsPermission.getUMask(conf)),
initialPermission);
+
assertEquals(FsPermission.getFileDefault().applyUMask(FsPermission.getUMask(conf)),
initialPermission);
}
catch(Exception e) {
System.out.println(StringUtils.stringifyException(e));
Modified:
hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/test/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/test/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier?rev=1433133&r1=1433132&r2=1433133&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/test/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier
(original)
+++
hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/test/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier
Mon Jan 14 21:12:16 2013
@@ -1,2 +1,15 @@
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
org.apache.hadoop.ipc.TestSaslRPC$TestTokenIdentifier
org.apache.hadoop.security.token.delegation.TestDelegationToken$TestDelegationTokenIdentifier
Modified:
hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/test/resources/kdc/killKdc.sh
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/test/resources/kdc/killKdc.sh?rev=1433133&r1=1433132&r2=1433133&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/test/resources/kdc/killKdc.sh
(original)
+++
hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/test/resources/kdc/killKdc.sh
Mon Jan 14 21:12:16 2013
@@ -1,3 +1,19 @@
#!/bin/sh
-ps -ef | grep apacheds | grep -v grep | cut -f4 -d ' ' |xargs kill -9
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+ps -ef | grep apacheds | grep -v grep | awk '{printf $2"\n"}' | xargs -t
--no-run-if-empty kill -9
Modified: hadoop/common/branches/HDFS-347/hadoop-common-project/pom.xml
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-common-project/pom.xml?rev=1433133&r1=1433132&r2=1433133&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-common-project/pom.xml (original)
+++ hadoop/common/branches/HDFS-347/hadoop-common-project/pom.xml Mon Jan 14
21:12:16 2013
@@ -49,9 +49,6 @@
<groupId>org.apache.rat</groupId>
<artifactId>apache-rat-plugin</artifactId>
<configuration>
- <includes>
- <include>pom.xml</include>
- </includes>
</configuration>
</plugin>
</plugins>
