Author: aching
Date: Wed Aug 31 16:38:31 2011
New Revision: 1163689
URL: http://svn.apache.org/viewvc?rev=1163689&view=rev
Log:
GIRAPH-14. Support for the Facebook Hadoop branch. (aching)
Modified:
incubator/giraph/trunk/CHANGELOG
incubator/giraph/trunk/README
incubator/giraph/trunk/pom.xml
incubator/giraph/trunk/src/main/java/org/apache/giraph/comm/BasicRPCCommunications.java
incubator/giraph/trunk/src/main/java/org/apache/giraph/comm/CommunicationsInterface.java
incubator/giraph/trunk/src/main/java/org/apache/giraph/comm/RPCCommunications.java
Modified: incubator/giraph/trunk/CHANGELOG
URL:
http://svn.apache.org/viewvc/incubator/giraph/trunk/CHANGELOG?rev=1163689&r1=1163688&r2=1163689&view=diff
==============================================================================
--- incubator/giraph/trunk/CHANGELOG (original)
+++ incubator/giraph/trunk/CHANGELOG Wed Aug 31 16:38:31 2011
@@ -1,5 +1,6 @@
Giraph Change Log
Release 0.70.0 - unreleased
-
+
+ GIRAPH-14. Support for the Facebook Hadoop branch. (aching)
GIRAPH-16. Add Apache RAT to the verify build step. (omalley)
Modified: incubator/giraph/trunk/README
URL:
http://svn.apache.org/viewvc/incubator/giraph/trunk/README?rev=1163689&r1=1163688&r2=1163689&view=diff
==============================================================================
--- incubator/giraph/trunk/README (original)
+++ incubator/giraph/trunk/README Wed Aug 31 16:38:31 2011
@@ -42,9 +42,13 @@ Secure Hadoop versions:
Unsecure Hadoop versions:
- Apache Hadoop 0.20.1, 0.20.2, 0.20.3
-While we provide support for unsecure Hadoop with the maven profile
-'hadoop_non_secure', we have been primarily focusing on secure Hadoop
-releases at this time.
+Facebook Hadoop release (https://github.com/facebook/hadoop-20-warehouse):
+- GitHub master
+
+While we provide support for the unsecure and Facebook versions of Hadoop
+with the maven profiles 'hadoop_non_secure' and 'hadoop_facebook',
+respectively, we have been primarily focusing on secure Hadoop releases
+at this time.
-------------------------------
@@ -63,6 +67,11 @@ Use the maven commands with secure Hadoo
For the non-secure versions of Hadoop, run the maven commands with the
additional argument '-Dhadoop=non_secure' to enable the maven profile
- 'hadoop_non_secure'. An example compilation command is
+'hadoop_non_secure'. An example compilation command is
'mvn -Dhadoop=non_secure compile'.
+For the Facebook Hadoop release, run the maven commands with the
+additional arguments '-Dhadoop=facebook' to enable the maven profile
+'hadoop_facebook' as well as a location for the hadoop core jar file. An
+example compilation command is 'mvn -Dhadoop=facebook
+-Dhadoop.jar.path=/tmp/hadoop-0.20.1-core.jar compile'.
Modified: incubator/giraph/trunk/pom.xml
URL:
http://svn.apache.org/viewvc/incubator/giraph/trunk/pom.xml?rev=1163689&r1=1163688&r2=1163689&view=diff
==============================================================================
--- incubator/giraph/trunk/pom.xml (original)
+++ incubator/giraph/trunk/pom.xml Wed Aug 31 16:38:31 2011
@@ -152,7 +152,7 @@ under the License.
<artifactId>maven-assembly-plugin</artifactId>
<version>2.2</version>
<configuration>
- <outputDirectory>target</outputDirectory>
+ <outputDirectory>target</outputDirectory>
<descriptorRefs>
<descriptorRef>jar-with-dependencies</descriptorRef>
</descriptorRefs>
@@ -183,7 +183,7 @@ under the License.
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
- <version>2.3.2</version>
+ <version>2.3.2</version>
<configuration>
<source>1.6</source>
<target>1.6</target>
@@ -232,12 +232,11 @@ under the License.
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-javadoc-plugin</artifactId>
<version>2.7</version>
- </plugin>
+ </plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-checkstyle-plugin</artifactId>
<version>2.6</version>
-
</plugin>
<plugin>
<groupId>org.codehaus.mojo</groupId>
@@ -342,6 +341,66 @@ under the License.
</plugins>
</build>
</profile>
+ <profile>
+ <id>hadoop_facebook</id>
+ <activation>
+ <property>
+ <name>hadoop</name>
+ <value>facebook</value>
+ </property>
+ </activation>
+ <dependencies>
+ <dependency>
+ <groupId>com.facebook.hadoop</groupId>
+ <artifactId>hadoop-core</artifactId>
+ <version>0.20.1</version>
+ <type>jar</type>
+ <scope>system</scope>
+ <systemPath>${hadoop.jar.path}</systemPath>
+ </dependency>
+ </dependencies>
+ <build>
+ <resources>
+ <resource>
+ <directory>src/main/java/org/apache/giraph/hadoop</directory>
+ <excludes>
+ <exclude>BspTokenSelector.java</exclude>
+ </excludes>
+ </resource>
+ </resources>
+ <plugins>
+ <plugin>
+ <groupId>org.sonatype.plugins</groupId>
+ <artifactId>munge-maven-plugin</artifactId>
+ <version>1.0</version>
+ <executions>
+ <execution>
+ <id>munge</id>
+ <phase>generate-sources</phase>
+ <goals>
+ <goal>munge</goal>
+ </goals>
+ </execution>
+ </executions>
+ <configuration>
+ <symbols>HADOOP_FACEBOOK</symbols>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <version>2.3.2</version>
+ <configuration>
+ <excludes>
+ <exclude>**/BspTokenSelector.java</exclude>
+ </excludes>
+ <source>1.6</source>
+ <target>1.6</target>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
</profiles>
<dependencies>
@@ -361,8 +420,8 @@ under the License.
<groupId>org.codehaus.jackson</groupId>
<artifactId>jackson-core-asl</artifactId>
<version>${jackson.version}</version>
- </dependency>
- <dependency>
+ </dependency>
+ <dependency>
<groupId>org.codehaus.jackson</groupId>
<artifactId>jackson-mapper-asl</artifactId>
<version>${jackson.version}</version>
@@ -385,18 +444,23 @@ under the License.
<artifactId>jms</artifactId>
</exclusion>
</exclusions>
- </dependency>
- <dependency>
+ </dependency>
+ <dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-io</artifactId>
<version>1.3.2</version>
</dependency>
- <dependency>
+ <dependency>
+ <groupId>commons-cli</groupId>
+ <artifactId>commons-cli</artifactId>
+ <version>1.2</version>
+ </dependency>
+ <dependency>
<groupId>net.iharder</groupId>
<artifactId>base64</artifactId>
<version>2.3.8</version>
</dependency>
- <dependency>
+ <dependency>
<groupId>org.json</groupId>
<artifactId>json</artifactId>
<version>20090211</version>
Modified:
incubator/giraph/trunk/src/main/java/org/apache/giraph/comm/BasicRPCCommunications.java
URL:
http://svn.apache.org/viewvc/incubator/giraph/trunk/src/main/java/org/apache/giraph/comm/BasicRPCCommunications.java?rev=1163689&r1=1163688&r2=1163689&view=diff
==============================================================================
---
incubator/giraph/trunk/src/main/java/org/apache/giraph/comm/BasicRPCCommunications.java
(original)
+++
incubator/giraph/trunk/src/main/java/org/apache/giraph/comm/BasicRPCCommunications.java
Wed Aug 31 16:38:31 2011
@@ -50,6 +50,10 @@ import org.apache.giraph.graph.VertexRes
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableComparable;
+
+/*if[HADOOP_FACEBOOK]
+import org.apache.hadoop.ipc.ProtocolSignature;
+end[HADOOP_FACEBOOK]*/
import org.apache.hadoop.ipc.RPC;
import org.apache.hadoop.ipc.RPC.Server;
import org.apache.hadoop.mapreduce.Mapper;
@@ -557,6 +561,15 @@ public abstract class BasicRPCCommunicat
return versionID;
}
+/*if[HADOOP_FACEBOOK]
+ public ProtocolSignature getProtocolSignature(
+ String protocol,
+ long clientVersion,
+ int clientMethodsHash) throws IOException {
+ return new ProtocolSignature(versionID, null);
+ }
+end[HADOOP_FACEBOOK]*/
+
@Override
public void closeConnections() throws IOException {
for (PeerThread pt : peerThreads.values()) {
@@ -572,6 +585,7 @@ public abstract class BasicRPCCommunicat
}
}
+
@Override
public final void close() {
LOG.info("close: shutting down RPC server");
Modified:
incubator/giraph/trunk/src/main/java/org/apache/giraph/comm/CommunicationsInterface.java
URL:
http://svn.apache.org/viewvc/incubator/giraph/trunk/src/main/java/org/apache/giraph/comm/CommunicationsInterface.java?rev=1163689&r1=1163688&r2=1163689&view=diff
==============================================================================
---
incubator/giraph/trunk/src/main/java/org/apache/giraph/comm/CommunicationsInterface.java
(original)
+++
incubator/giraph/trunk/src/main/java/org/apache/giraph/comm/CommunicationsInterface.java
Wed Aug 31 16:38:31 2011
@@ -23,11 +23,11 @@ import java.io.IOException;
import org.apache.giraph.graph.Edge;
import org.apache.giraph.graph.MutableVertex;
import org.apache.giraph.graph.VertexRange;
-/*if[HADOOP_NON_SECURE]
- else[HADOOP_NON_SECURE]*/
+/*if_not[HADOOP]
+ else[HADOOP]*/
import org.apache.giraph.hadoop.BspTokenSelector;
import org.apache.hadoop.security.token.TokenInfo;
-/*end[HADOOP_NON_SECURE]*/
+/*end[HADOOP]*/
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableComparable;
import org.apache.hadoop.ipc.VersionedProtocol;
@@ -41,10 +41,10 @@ import org.apache.hadoop.ipc.VersionedPr
*
**/
@SuppressWarnings("rawtypes")
-/*if[HADOOP_NON_SECURE]
- else[HADOOP_NON_SECURE]*/
+/*if_not[HADOOP]
+ else[HADOOP]*/
@TokenInfo(BspTokenSelector.class)
-/*end[HADOOP_NON_SECURE]*/
+/*end[HADOOP]*/
public interface CommunicationsInterface<
I extends WritableComparable,
V extends Writable,
Modified:
incubator/giraph/trunk/src/main/java/org/apache/giraph/comm/RPCCommunications.java
URL:
http://svn.apache.org/viewvc/incubator/giraph/trunk/src/main/java/org/apache/giraph/comm/RPCCommunications.java?rev=1163689&r1=1163688&r2=1163689&view=diff
==============================================================================
---
incubator/giraph/trunk/src/main/java/org/apache/giraph/comm/RPCCommunications.java
(original)
+++
incubator/giraph/trunk/src/main/java/org/apache/giraph/comm/RPCCommunications.java
Wed Aug 31 16:38:31 2011
@@ -23,8 +23,8 @@ import java.io.IOException;
import java.net.InetSocketAddress;
import java.net.UnknownHostException;
-/*if[HADOOP_NON_SECURE]
-else[HADOOP_NON_SECURE]*/
+/*if_not[HADOOP]
+else[HADOOP]*/
import java.security.PrivilegedExceptionAction;
import org.apache.hadoop.mapreduce.security.TokenCache;
import org.apache.hadoop.mapreduce.security.token.JobTokenIdentifier;
@@ -33,7 +33,7 @@ import org.apache.hadoop.security.Creden
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.authorize.ServiceAuthorizationManager;
import org.apache.hadoop.security.token.Token;
-/*end[HADOOP_NON_SECURE]*/
+/*end[HADOOP]*/
import org.apache.log4j.Logger;
@@ -53,11 +53,11 @@ public class RPCCommunications<
V extends Writable,
E extends Writable,
M extends Writable>
-/*if[HADOOP_NON_SECURE]
+/*if_not[HADOOP]
extends BasicRPCCommunications<I, V, E, M, Object> {
-else[HADOOP_NON_SECURE]*/
+else[HADOOP]*/
extends BasicRPCCommunications<I, V, E, M, Token<JobTokenIdentifier>> {
-/*end[HADOOP_NON_SECURE]*/
+/*end[HADOOP]*/
/** Class logger */
public static final Logger LOG = Logger.getLogger(RPCCommunications.class);
@@ -68,11 +68,11 @@ else[HADOOP_NON_SECURE]*/
super(context, service);
}
-/*if[HADOOP_NON_SECURE]
+/*if_not[HADOOP]
protected Object createJobToken() throws IOException {
return null;
}
-else[HADOOP_NON_SECURE]*/
+else[HADOOP]*/
protected Token<JobTokenIdentifier> createJobToken() throws IOException {
String localJobTokenFile = System.getenv().get(
UserGroupInformation.HADOOP_TOKEN_FILE_LOCATION);
@@ -83,16 +83,16 @@ else[HADOOP_NON_SECURE]*/
}
return null;
}
-/*end[HADOOP_NON_SECURE]*/
+/*end[HADOOP]*/
protected Server getRPCServer(
InetSocketAddress myAddress, int numHandlers, String jobId,
-/*if[HADOOP_NON_SECURE]
+/*if_not[HADOOP]
Object jt) throws IOException {
return RPC.getServer(this, myAddress.getHostName(),
myAddress.getPort(),
numHandlers, false, conf);
}
-else[HADOOP_NON_SECURE]*/
+else[HADOOP]*/
Token<JobTokenIdentifier> jt) throws IOException {
@SuppressWarnings("deprecation")
String hadoopSecurityAuthorization =
@@ -113,26 +113,26 @@ else[HADOOP_NON_SECURE]*/
return RPC.getServer(this, myAddress.getHostName(),
myAddress.getPort(),
numHandlers, false, conf, jobTokenSecretManager);
}
-/*end[HADOOP_NON_SECURE]*/
+/*end[HADOOP]*/
protected CommunicationsInterface<I, V, E, M> getRPCProxy(
final InetSocketAddress addr,
String jobId,
-/*if[HADOOP_NON_SECURE]
+/*if_not[HADOOP]
Object jt)
-else[HADOOP_NON_SECURE]*/
+else[HADOOP]*/
Token<JobTokenIdentifier> jt)
-/*end[HADOOP_NON_SECURE]*/
+/*end[HADOOP]*/
throws IOException, InterruptedException {
final Configuration config = new Configuration(conf);
-/*if[HADOOP_NON_SECURE]
+/*if_not[HADOOP]
@SuppressWarnings("unchecked")
CommunicationsInterface<I, V, E, M> proxy =
(CommunicationsInterface<I, V, E, M>)RPC.getProxy(
CommunicationsInterface.class, versionID, addr, config);
return proxy;
-else[HADOOP_NON_SECURE]*/
+else[HADOOP]*/
if (jt == null) {
@SuppressWarnings("unchecked")
CommunicationsInterface<I, V, E, M> proxy =
@@ -159,6 +159,6 @@ else[HADOOP_NON_SECURE]*/
}
});
return proxy;
-/*end[HADOOP_NON_SECURE]*/
+/*end[HADOOP]*/
}
}
