Author: aching
Date: Wed Aug 31 16:38:31 2011
New Revision: 1163689

URL: http://svn.apache.org/viewvc?rev=1163689&view=rev
Log:
GIRAPH-14. Support for the Facebook Hadoop branch. (aching)


Modified:
    incubator/giraph/trunk/CHANGELOG
    incubator/giraph/trunk/README
    incubator/giraph/trunk/pom.xml
    
incubator/giraph/trunk/src/main/java/org/apache/giraph/comm/BasicRPCCommunications.java
    
incubator/giraph/trunk/src/main/java/org/apache/giraph/comm/CommunicationsInterface.java
    
incubator/giraph/trunk/src/main/java/org/apache/giraph/comm/RPCCommunications.java

Modified: incubator/giraph/trunk/CHANGELOG
URL: 
http://svn.apache.org/viewvc/incubator/giraph/trunk/CHANGELOG?rev=1163689&r1=1163688&r2=1163689&view=diff
==============================================================================
--- incubator/giraph/trunk/CHANGELOG (original)
+++ incubator/giraph/trunk/CHANGELOG Wed Aug 31 16:38:31 2011
@@ -1,5 +1,6 @@
 Giraph Change Log
 
 Release 0.70.0 - unreleased
-
+  
+  GIRAPH-14. Support for the Facebook Hadoop branch. (aching)
   GIRAPH-16. Add Apache RAT to the verify build step. (omalley)

Modified: incubator/giraph/trunk/README
URL: 
http://svn.apache.org/viewvc/incubator/giraph/trunk/README?rev=1163689&r1=1163688&r2=1163689&view=diff
==============================================================================
--- incubator/giraph/trunk/README (original)
+++ incubator/giraph/trunk/README Wed Aug 31 16:38:31 2011
@@ -42,9 +42,13 @@ Secure Hadoop versions:
 Unsecure Hadoop versions:
 - Apache Hadoop 0.20.1, 0.20.2, 0.20.3
 
-While we provide support for unsecure Hadoop with the maven profile
-'hadoop_non_secure', we have been primarily focusing on secure Hadoop
-releases at this time.
+Facebook Hadoop release (https://github.com/facebook/hadoop-20-warehouse):
+- GitHub master
+
+While we provide support for the unsecure and Facebook versions of Hadoop
+with the maven profiles 'hadoop_non_secure' and 'hadoop_facebook',
+respectively, we have been primarily focusing on secure Hadoop releases
+at this time.
 
 -------------------------------
 
@@ -63,6 +67,11 @@ Use the maven commands with secure Hadoo
 
 For the non-secure versions of Hadoop, run the maven commands with the
 additional argument '-Dhadoop=non_secure' to enable the maven profile
- 'hadoop_non_secure'.  An example compilation command is
+'hadoop_non_secure'.  An example compilation command is
 'mvn -Dhadoop=non_secure compile'.
 
+For the Facebook Hadoop release, run the maven commands with the
+additional arguments '-Dhadoop=facebook' to enable the maven profile
+'hadoop_facebook' as well as a location for the hadoop core jar file.  An
+example compilation command is 'mvn -Dhadoop=facebook
+-Dhadoop.jar.path=/tmp/hadoop-0.20.1-core.jar compile'.

Modified: incubator/giraph/trunk/pom.xml
URL: 
http://svn.apache.org/viewvc/incubator/giraph/trunk/pom.xml?rev=1163689&r1=1163688&r2=1163689&view=diff
==============================================================================
--- incubator/giraph/trunk/pom.xml (original)
+++ incubator/giraph/trunk/pom.xml Wed Aug 31 16:38:31 2011
@@ -152,7 +152,7 @@ under the License.
         <artifactId>maven-assembly-plugin</artifactId>
         <version>2.2</version>
         <configuration>
-         <outputDirectory>target</outputDirectory>
+         <outputDirectory>target</outputDirectory>
           <descriptorRefs>
             <descriptorRef>jar-with-dependencies</descriptorRef>
           </descriptorRefs>
@@ -183,7 +183,7 @@ under the License.
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-compiler-plugin</artifactId>
-               <version>2.3.2</version>
+        <version>2.3.2</version>
         <configuration>
           <source>1.6</source>
           <target>1.6</target>
@@ -232,12 +232,11 @@ under the License.
             <groupId>org.apache.maven.plugins</groupId>
             <artifactId>maven-javadoc-plugin</artifactId>
             <version>2.7</version>
-          </plugin>     
+          </plugin>
           <plugin>
             <groupId>org.apache.maven.plugins</groupId>
             <artifactId>maven-checkstyle-plugin</artifactId>
             <version>2.6</version>
-
           </plugin>
           <plugin>
             <groupId>org.codehaus.mojo</groupId>
@@ -342,6 +341,66 @@ under the License.
         </plugins>
       </build>
     </profile>
+    <profile>
+      <id>hadoop_facebook</id>
+       <activation>
+        <property>
+          <name>hadoop</name>
+          <value>facebook</value>
+        </property>
+      </activation>
+      <dependencies>
+        <dependency>
+          <groupId>com.facebook.hadoop</groupId>
+          <artifactId>hadoop-core</artifactId>
+          <version>0.20.1</version>
+          <type>jar</type>
+          <scope>system</scope>
+          <systemPath>${hadoop.jar.path}</systemPath>
+        </dependency>
+      </dependencies>
+      <build>
+        <resources>
+          <resource>
+            <directory>src/main/java/org/apache/giraph/hadoop</directory>
+            <excludes>
+              <exclude>BspTokenSelector.java</exclude>
+            </excludes>
+          </resource>
+        </resources>
+        <plugins>
+          <plugin>
+            <groupId>org.sonatype.plugins</groupId>
+            <artifactId>munge-maven-plugin</artifactId>
+            <version>1.0</version>
+            <executions>
+              <execution>
+                <id>munge</id>
+                <phase>generate-sources</phase>
+                <goals>
+                  <goal>munge</goal>
+                </goals>
+              </execution>
+            </executions>
+            <configuration>
+              <symbols>HADOOP_FACEBOOK</symbols>
+            </configuration>
+          </plugin>
+          <plugin>
+            <groupId>org.apache.maven.plugins</groupId>
+            <artifactId>maven-compiler-plugin</artifactId>
+            <version>2.3.2</version>
+            <configuration>
+              <excludes>
+                <exclude>**/BspTokenSelector.java</exclude>
+              </excludes>
+              <source>1.6</source>
+              <target>1.6</target>
+            </configuration>
+          </plugin>
+        </plugins>
+      </build>
+    </profile>
   </profiles>
 
   <dependencies>
@@ -361,8 +420,8 @@ under the License.
       <groupId>org.codehaus.jackson</groupId>
       <artifactId>jackson-core-asl</artifactId>
       <version>${jackson.version}</version>
-       </dependency>
-       <dependency>
+    </dependency>
+    <dependency>
       <groupId>org.codehaus.jackson</groupId>
       <artifactId>jackson-mapper-asl</artifactId>
       <version>${jackson.version}</version>
@@ -385,18 +444,23 @@ under the License.
           <artifactId>jms</artifactId>
         </exclusion>
       </exclusions>
-       </dependency>
-       <dependency>
+    </dependency>
+    <dependency>
       <groupId>org.apache.commons</groupId>
       <artifactId>commons-io</artifactId>
       <version>1.3.2</version>
     </dependency>
-       <dependency>
+    <dependency>
+      <groupId>commons-cli</groupId>
+      <artifactId>commons-cli</artifactId>
+      <version>1.2</version>
+    </dependency>
+    <dependency>
       <groupId>net.iharder</groupId>
       <artifactId>base64</artifactId>
       <version>2.3.8</version>
     </dependency>
-       <dependency>
+    <dependency>
       <groupId>org.json</groupId>
       <artifactId>json</artifactId>
       <version>20090211</version>

Modified: 
incubator/giraph/trunk/src/main/java/org/apache/giraph/comm/BasicRPCCommunications.java
URL: 
http://svn.apache.org/viewvc/incubator/giraph/trunk/src/main/java/org/apache/giraph/comm/BasicRPCCommunications.java?rev=1163689&r1=1163688&r2=1163689&view=diff
==============================================================================
--- 
incubator/giraph/trunk/src/main/java/org/apache/giraph/comm/BasicRPCCommunications.java
 (original)
+++ 
incubator/giraph/trunk/src/main/java/org/apache/giraph/comm/BasicRPCCommunications.java
 Wed Aug 31 16:38:31 2011
@@ -50,6 +50,10 @@ import org.apache.giraph.graph.VertexRes
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.io.Writable;
 import org.apache.hadoop.io.WritableComparable;
+
+/*if[HADOOP_FACEBOOK]
+import org.apache.hadoop.ipc.ProtocolSignature;
+end[HADOOP_FACEBOOK]*/
 import org.apache.hadoop.ipc.RPC;
 import org.apache.hadoop.ipc.RPC.Server;
 import org.apache.hadoop.mapreduce.Mapper;
@@ -557,6 +561,15 @@ public abstract class BasicRPCCommunicat
         return versionID;
     }
 
+/*if[HADOOP_FACEBOOK]
+    public ProtocolSignature getProtocolSignature(
+            String protocol,
+            long clientVersion,
+            int clientMethodsHash) throws IOException {
+        return new ProtocolSignature(versionID, null);
+    }
+end[HADOOP_FACEBOOK]*/
+
     @Override
     public void closeConnections() throws IOException {
         for (PeerThread pt : peerThreads.values()) {
@@ -572,6 +585,7 @@ public abstract class BasicRPCCommunicat
         }
     }
 
+
     @Override
     public final void close() {
         LOG.info("close: shutting down RPC server");

Modified: 
incubator/giraph/trunk/src/main/java/org/apache/giraph/comm/CommunicationsInterface.java
URL: 
http://svn.apache.org/viewvc/incubator/giraph/trunk/src/main/java/org/apache/giraph/comm/CommunicationsInterface.java?rev=1163689&r1=1163688&r2=1163689&view=diff
==============================================================================
--- 
incubator/giraph/trunk/src/main/java/org/apache/giraph/comm/CommunicationsInterface.java
 (original)
+++ 
incubator/giraph/trunk/src/main/java/org/apache/giraph/comm/CommunicationsInterface.java
 Wed Aug 31 16:38:31 2011
@@ -23,11 +23,11 @@ import java.io.IOException;
 import org.apache.giraph.graph.Edge;
 import org.apache.giraph.graph.MutableVertex;
 import org.apache.giraph.graph.VertexRange;
-/*if[HADOOP_NON_SECURE]
- else[HADOOP_NON_SECURE]*/
+/*if_not[HADOOP]
+ else[HADOOP]*/
 import org.apache.giraph.hadoop.BspTokenSelector;
 import org.apache.hadoop.security.token.TokenInfo;
-/*end[HADOOP_NON_SECURE]*/
+/*end[HADOOP]*/
 import org.apache.hadoop.io.Writable;
 import org.apache.hadoop.io.WritableComparable;
 import org.apache.hadoop.ipc.VersionedProtocol;
@@ -41,10 +41,10 @@ import org.apache.hadoop.ipc.VersionedPr
  *
  **/
 @SuppressWarnings("rawtypes")
-/*if[HADOOP_NON_SECURE]
- else[HADOOP_NON_SECURE]*/
+/*if_not[HADOOP]
+ else[HADOOP]*/
 @TokenInfo(BspTokenSelector.class)
-/*end[HADOOP_NON_SECURE]*/
+/*end[HADOOP]*/
 public interface CommunicationsInterface<
         I extends WritableComparable,
         V extends Writable,

Modified: 
incubator/giraph/trunk/src/main/java/org/apache/giraph/comm/RPCCommunications.java
URL: 
http://svn.apache.org/viewvc/incubator/giraph/trunk/src/main/java/org/apache/giraph/comm/RPCCommunications.java?rev=1163689&r1=1163688&r2=1163689&view=diff
==============================================================================
--- 
incubator/giraph/trunk/src/main/java/org/apache/giraph/comm/RPCCommunications.java
 (original)
+++ 
incubator/giraph/trunk/src/main/java/org/apache/giraph/comm/RPCCommunications.java
 Wed Aug 31 16:38:31 2011
@@ -23,8 +23,8 @@ import java.io.IOException;
 import java.net.InetSocketAddress;
 import java.net.UnknownHostException;
 
-/*if[HADOOP_NON_SECURE]
-else[HADOOP_NON_SECURE]*/
+/*if_not[HADOOP]
+else[HADOOP]*/
 import java.security.PrivilegedExceptionAction;
 import org.apache.hadoop.mapreduce.security.TokenCache;
 import org.apache.hadoop.mapreduce.security.token.JobTokenIdentifier;
@@ -33,7 +33,7 @@ import org.apache.hadoop.security.Creden
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.security.authorize.ServiceAuthorizationManager;
 import org.apache.hadoop.security.token.Token;
-/*end[HADOOP_NON_SECURE]*/
+/*end[HADOOP]*/
 
 import org.apache.log4j.Logger;
 
@@ -53,11 +53,11 @@ public class RPCCommunications<
         V extends Writable,
         E extends Writable,
         M extends Writable>
-/*if[HADOOP_NON_SECURE]
+/*if_not[HADOOP]
 extends BasicRPCCommunications<I, V, E, M, Object> {
-else[HADOOP_NON_SECURE]*/
+else[HADOOP]*/
         extends BasicRPCCommunications<I, V, E, M, Token<JobTokenIdentifier>> {
-/*end[HADOOP_NON_SECURE]*/
+/*end[HADOOP]*/
 
     /** Class logger */
     public static final Logger LOG = Logger.getLogger(RPCCommunications.class);
@@ -68,11 +68,11 @@ else[HADOOP_NON_SECURE]*/
         super(context, service);
     }
 
-/*if[HADOOP_NON_SECURE]
+/*if_not[HADOOP]
     protected Object createJobToken() throws IOException {
         return null;
     }
-else[HADOOP_NON_SECURE]*/
+else[HADOOP]*/
     protected Token<JobTokenIdentifier> createJobToken() throws IOException {
         String localJobTokenFile = System.getenv().get(
                 UserGroupInformation.HADOOP_TOKEN_FILE_LOCATION);
@@ -83,16 +83,16 @@ else[HADOOP_NON_SECURE]*/
         }
         return null;
     }
-/*end[HADOOP_NON_SECURE]*/
+/*end[HADOOP]*/
 
     protected Server getRPCServer(
             InetSocketAddress myAddress, int numHandlers, String jobId,
-/*if[HADOOP_NON_SECURE]
+/*if_not[HADOOP]
             Object jt) throws IOException {
         return RPC.getServer(this, myAddress.getHostName(), 
myAddress.getPort(),
             numHandlers, false, conf);
     }
-else[HADOOP_NON_SECURE]*/
+else[HADOOP]*/
             Token<JobTokenIdentifier> jt) throws IOException {
         @SuppressWarnings("deprecation")
         String hadoopSecurityAuthorization =
@@ -113,26 +113,26 @@ else[HADOOP_NON_SECURE]*/
         return RPC.getServer(this, myAddress.getHostName(), 
myAddress.getPort(),
                 numHandlers, false, conf, jobTokenSecretManager);
     }
-/*end[HADOOP_NON_SECURE]*/
+/*end[HADOOP]*/
 
     protected CommunicationsInterface<I, V, E, M> getRPCProxy(
             final InetSocketAddress addr,
             String jobId,
-/*if[HADOOP_NON_SECURE]
+/*if_not[HADOOP]
             Object jt)
-else[HADOOP_NON_SECURE]*/
+else[HADOOP]*/
             Token<JobTokenIdentifier> jt)
-/*end[HADOOP_NON_SECURE]*/
+/*end[HADOOP]*/
             throws IOException, InterruptedException {
         final Configuration config = new Configuration(conf);
 
-/*if[HADOOP_NON_SECURE]
+/*if_not[HADOOP]
         @SuppressWarnings("unchecked")
         CommunicationsInterface<I, V, E, M> proxy =
             (CommunicationsInterface<I, V, E, M>)RPC.getProxy(
                  CommunicationsInterface.class, versionID, addr, config);
         return proxy;
-else[HADOOP_NON_SECURE]*/
+else[HADOOP]*/
         if (jt == null) {
             @SuppressWarnings("unchecked")
             CommunicationsInterface<I, V, E, M> proxy =
@@ -159,6 +159,6 @@ else[HADOOP_NON_SECURE]*/
             }
         });
                return proxy;
-/*end[HADOOP_NON_SECURE]*/
+/*end[HADOOP]*/
     }
 }


Reply via email to