Author: gates
Date: Fri Feb 22 15:39:11 2008
New Revision: 630355

URL: http://svn.apache.org/viewvc?rev=630355&view=rev
Log:
PIG-72: Move unit tests to use MiniDFS and MiniMR so that unit tests can be run 
w/o access to a hadoop cluster.


Added:
    incubator/pig/trunk/test/org/apache/pig/test/MiniCluster.java
Modified:
    incubator/pig/trunk/CHANGES.txt
    incubator/pig/trunk/build.xml
    incubator/pig/trunk/lib/hadoop15.jar
    incubator/pig/trunk/test/org/apache/pig/test/TestAlgebraicEval.java
    incubator/pig/trunk/test/org/apache/pig/test/TestCompressedFiles.java
    incubator/pig/trunk/test/org/apache/pig/test/TestEvalPipeline.java
    incubator/pig/trunk/test/org/apache/pig/test/TestFilterOpNumeric.java
    incubator/pig/trunk/test/org/apache/pig/test/TestFilterOpString.java
    incubator/pig/trunk/test/org/apache/pig/test/TestGrunt.java
    incubator/pig/trunk/test/org/apache/pig/test/TestInfixArithmetic.java
    incubator/pig/trunk/test/org/apache/pig/test/TestLargeFile.java
    incubator/pig/trunk/test/org/apache/pig/test/TestMapReduce.java
    incubator/pig/trunk/test/org/apache/pig/test/TestOrderBy.java
    incubator/pig/trunk/test/org/apache/pig/test/TestPi.java
    incubator/pig/trunk/test/org/apache/pig/test/TestPigServer.java
    incubator/pig/trunk/test/org/apache/pig/test/TestPigSplit.java
    incubator/pig/trunk/test/org/apache/pig/test/TestStore.java

Modified: incubator/pig/trunk/CHANGES.txt
URL: 
http://svn.apache.org/viewvc/incubator/pig/trunk/CHANGES.txt?rev=630355&r1=630354&r2=630355&view=diff
==============================================================================
--- incubator/pig/trunk/CHANGES.txt (original)
+++ incubator/pig/trunk/CHANGES.txt Fri Feb 22 15:39:11 2008
@@ -122,3 +122,6 @@
     olgan)
 
     PIG-109: improved exception handling (oae via olgan)
+
+       PIG-72: Move unit tests to use MiniDFS and MiniMR so that unit tests can
+       be run w/o access to a hadoop cluster. (xuzh via gates)

Modified: incubator/pig/trunk/build.xml
URL: 
http://svn.apache.org/viewvc/incubator/pig/trunk/build.xml?rev=630355&r1=630354&r2=630355&view=diff
==============================================================================
--- incubator/pig/trunk/build.xml (original)
+++ incubator/pig/trunk/build.xml Fri Feb 22 15:39:11 2008
@@ -6,6 +6,7 @@
        <property name="src.dir" value="${basedir}/src" />
        <property name="doc.dir" value="${basedir}/doc" />
         <property name="test.reports.dir" value="${basedir}/test/reports" />
+    <property name="test.log.dir" value="${basedir}/test/logs"/>
        <property name="shock.src.dir" value="${basedir}/lib-src/shock" />
        <property name="bzip2.src.dir" value="${basedir}/lib-src/bzip2" />
        <property name="test.src.dir" value="${basedir}/test" />
@@ -16,7 +17,7 @@
        <property name="hod.command" value=""/>
        <property name="hod.param" value=""/>
        <property name="test.junit.output.format" value="plain"/>
-       <property name="junit.hadoop.conf" value=""/>
+       <property name="junit.hadoop.conf" value="${user.home}/pigtest/conf/"/>
                
        <!-- setup the classpath -->
        <path id="classpath">
@@ -105,6 +106,8 @@
        </target>
 
        <target name="test">
+           <delete dir="${test.log.dir}"/>
+           <mkdir dir="${test.log.dir}"/>
                <mkdir dir="test/reports"/>
                 <junit printsummary="on" haltonfailure="no"
                        errorProperty="tests.failed" 
failureProperty="tests.failed">
@@ -113,6 +116,7 @@
                        <sysproperty key="hod.server" value="${hod.server}"/>
                        <!-- <sysproperty key="hod.command" 
value="${hod.command}"/>
                        <sysproperty key="hod.param" value="${hod.param}"/> -->
+                   <sysproperty key="hadoop.log.dir" value="${test.log.dir}"/>
                        <classpath>
                                <pathelement location="${output.jarfile}"/>
                                <pathelement location="${junit.hadoop.conf}"/>

Modified: incubator/pig/trunk/lib/hadoop15.jar
URL: 
http://svn.apache.org/viewvc/incubator/pig/trunk/lib/hadoop15.jar?rev=630355&r1=630354&r2=630355&view=diff
==============================================================================
Binary files - no diff available.

Added: incubator/pig/trunk/test/org/apache/pig/test/MiniCluster.java
URL: 
http://svn.apache.org/viewvc/incubator/pig/trunk/test/org/apache/pig/test/MiniCluster.java?rev=630355&view=auto
==============================================================================
--- incubator/pig/trunk/test/org/apache/pig/test/MiniCluster.java (added)
+++ incubator/pig/trunk/test/org/apache/pig/test/MiniCluster.java Fri Feb 22 
15:39:11 2008
@@ -0,0 +1,102 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.pig.test;
+
+import java.io.*;
+
+import org.apache.hadoop.dfs.MiniDFSCluster;
+import org.apache.hadoop.mapred.MiniMRCluster;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.fs.FileSystem;
+
+/**
+ * This class builds a single instance of itself with the Singleton 
+ * design pattern. While building the single instance, it sets up a 
+ * mini cluster that actually consists of a mini DFS cluster and a 
+ * mini MapReduce cluster on the local machine and also sets up the 
+ * environment for Pig to run on top of the mini cluster.
+ */
+public class MiniCluster {
+       private MiniDFSCluster m_dfs = null;
+       private MiniMRCluster m_mr = null;
+       private FileSystem m_fileSys = null;
+       private JobConf m_conf = null;
+       
+       private final static MiniCluster INSTANCE = new MiniCluster();
+       
+       private MiniCluster() {
+               setupMiniDfsAndMrClusters();
+       }
+       
+       private void setupMiniDfsAndMrClusters() {
+               try {
+                       final int dataNodes = 4;     // There will be 4 data 
nodes
+                       final int taskTrackers = 4;  // There will be 4 task 
tracker nodes
+                       Configuration config = new Configuration();
+                       
+            // Builds and starts the mini dfs and mapreduce clusters
+            m_dfs = new MiniDFSCluster(config, dataNodes, true, null);
+            m_fileSys = m_dfs.getFileSystem();
+            m_mr = new MiniMRCluster(taskTrackers, m_fileSys.getName(), 1);
+            
+            // Create the configuration hadoop-site.xml file
+            File conf_dir = new File(System.getProperty("user.home"), 
"pigtest/conf/");
+            conf_dir.mkdirs();
+            File conf_file = new File(conf_dir, "hadoop-site.xml");
+            
+            // Write the necessary config info to hadoop-site.xml
+            m_conf = m_mr.createJobConf();      
+            m_conf.setInt("mapred.submit.replication", 2);
+            m_conf.set("dfs.datanode.address", "0.0.0.0:0");
+            m_conf.set("dfs.datanode.http.address", "0.0.0.0:0");
+            m_conf.write(new FileOutputStream(conf_file));
+            
+            // Set the system properties needed by Pig
+            System.setProperty("cluster", m_conf.get("mapred.job.tracker"));
+            System.setProperty("namenode", m_conf.get("fs.default.name"));
+            System.setProperty("junit.hadoop.conf", conf_dir.getPath());
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+       }
+       
+    /**
+     * Returns the single instance of class MiniClusterBuilder that
+     * represents the resouces for a mini dfs cluster and a mini 
+     * mapreduce cluster. 
+     */
+       public static MiniCluster buildCluster() {
+               return INSTANCE;
+       }
+       
+       protected void finalize() {
+               shutdownMiniDfsAndMrClusters();
+       }
+       
+       private void shutdownMiniDfsAndMrClusters() {
+               try {
+                       if (m_fileSys != null) { m_fileSys.close(); }
+               } catch (IOException e) {
+               e.printStackTrace();
+               }
+               if (m_dfs != null) { m_dfs.shutdown(); }
+               if (m_mr != null) { m_mr.shutdown(); }          
+       }
+}

Modified: incubator/pig/trunk/test/org/apache/pig/test/TestAlgebraicEval.java
URL: 
http://svn.apache.org/viewvc/incubator/pig/trunk/test/org/apache/pig/test/TestAlgebraicEval.java?rev=630355&r1=630354&r2=630355&view=diff
==============================================================================
--- incubator/pig/trunk/test/org/apache/pig/test/TestAlgebraicEval.java 
(original)
+++ incubator/pig/trunk/test/org/apache/pig/test/TestAlgebraicEval.java Fri Feb 
22 15:39:11 2008
@@ -33,6 +33,7 @@
 public class TestAlgebraicEval extends TestCase {
     
        private String initString = "mapreduce";
+       MiniCluster cluster = MiniCluster.buildCluster();
     @Test
     public void testSimpleCount() throws Throwable {
         int LOOP_COUNT = 1024;

Modified: incubator/pig/trunk/test/org/apache/pig/test/TestCompressedFiles.java
URL: 
http://svn.apache.org/viewvc/incubator/pig/trunk/test/org/apache/pig/test/TestCompressedFiles.java?rev=630355&r1=630354&r2=630355&view=diff
==============================================================================
--- incubator/pig/trunk/test/org/apache/pig/test/TestCompressedFiles.java 
(original)
+++ incubator/pig/trunk/test/org/apache/pig/test/TestCompressedFiles.java Fri 
Feb 22 15:39:11 2008
@@ -35,6 +35,7 @@
 public class TestCompressedFiles extends TestCase {
     
     private final Log log = LogFactory.getLog(getClass());
+    MiniCluster cluster = MiniCluster.buildCluster();
 
     File datFile;
     File gzFile;

Modified: incubator/pig/trunk/test/org/apache/pig/test/TestEvalPipeline.java
URL: 
http://svn.apache.org/viewvc/incubator/pig/trunk/test/org/apache/pig/test/TestEvalPipeline.java?rev=630355&r1=630354&r2=630355&view=diff
==============================================================================
--- incubator/pig/trunk/test/org/apache/pig/test/TestEvalPipeline.java 
(original)
+++ incubator/pig/trunk/test/org/apache/pig/test/TestEvalPipeline.java Fri Feb 
22 15:39:11 2008
@@ -48,6 +48,7 @@
 public class TestEvalPipeline extends TestCase {
        
        String initString = "mapreduce";
+       MiniCluster cluster = MiniCluster.buildCluster();
        
        
        static public class MyBagFunction extends EvalFunc<DataBag>{

Modified: incubator/pig/trunk/test/org/apache/pig/test/TestFilterOpNumeric.java
URL: 
http://svn.apache.org/viewvc/incubator/pig/trunk/test/org/apache/pig/test/TestFilterOpNumeric.java?rev=630355&r1=630354&r2=630355&view=diff
==============================================================================
--- incubator/pig/trunk/test/org/apache/pig/test/TestFilterOpNumeric.java 
(original)
+++ incubator/pig/trunk/test/org/apache/pig/test/TestFilterOpNumeric.java Fri 
Feb 22 15:39:11 2008
@@ -38,6 +38,7 @@
 
     private static int LOOP_COUNT = 1024;
     private String initString = "mapreduce";
+    MiniCluster cluster = MiniCluster.buildCluster();
     
     @Test
     public void testNumericEq() throws Throwable {

Modified: incubator/pig/trunk/test/org/apache/pig/test/TestFilterOpString.java
URL: 
http://svn.apache.org/viewvc/incubator/pig/trunk/test/org/apache/pig/test/TestFilterOpString.java?rev=630355&r1=630354&r2=630355&view=diff
==============================================================================
--- incubator/pig/trunk/test/org/apache/pig/test/TestFilterOpString.java 
(original)
+++ incubator/pig/trunk/test/org/apache/pig/test/TestFilterOpString.java Fri 
Feb 22 15:39:11 2008
@@ -38,6 +38,7 @@
 
     private static int LOOP_COUNT = 1024;    
     private String initString = "mapreduce";
+    MiniCluster cluster = MiniCluster.buildCluster();
     
     @Test
     public void testStringEq() throws Throwable {

Modified: incubator/pig/trunk/test/org/apache/pig/test/TestGrunt.java
URL: 
http://svn.apache.org/viewvc/incubator/pig/trunk/test/org/apache/pig/test/TestGrunt.java?rev=630355&r1=630354&r2=630355&view=diff
==============================================================================
--- incubator/pig/trunk/test/org/apache/pig/test/TestGrunt.java (original)
+++ incubator/pig/trunk/test/org/apache/pig/test/TestGrunt.java Fri Feb 22 
15:39:11 2008
@@ -12,6 +12,7 @@
 import java.io.BufferedReader;
 
 public class TestGrunt extends TestCase {
+    MiniCluster cluster = MiniCluster.buildCluster();
 
        
        @Test 

Modified: incubator/pig/trunk/test/org/apache/pig/test/TestInfixArithmetic.java
URL: 
http://svn.apache.org/viewvc/incubator/pig/trunk/test/org/apache/pig/test/TestInfixArithmetic.java?rev=630355&r1=630354&r2=630355&view=diff
==============================================================================
--- incubator/pig/trunk/test/org/apache/pig/test/TestInfixArithmetic.java 
(original)
+++ incubator/pig/trunk/test/org/apache/pig/test/TestInfixArithmetic.java Fri 
Feb 22 15:39:11 2008
@@ -38,6 +38,7 @@
 
     private static int LOOP_COUNT = 1024;    
     private String initString = "mapreduce";
+    MiniCluster cluster = MiniCluster.buildCluster();
     
     @Test
     public void testAdd() throws Throwable {

Modified: incubator/pig/trunk/test/org/apache/pig/test/TestLargeFile.java
URL: 
http://svn.apache.org/viewvc/incubator/pig/trunk/test/org/apache/pig/test/TestLargeFile.java?rev=630355&r1=630354&r2=630355&view=diff
==============================================================================
--- incubator/pig/trunk/test/org/apache/pig/test/TestLargeFile.java (original)
+++ incubator/pig/trunk/test/org/apache/pig/test/TestLargeFile.java Fri Feb 22 
15:39:11 2008
@@ -52,6 +52,7 @@
        private int max_rand = 500;
        private String initString = "mapreduce";
 //     private double sum = 0.0, sumIn = 0.0;
+       MiniCluster cluster = MiniCluster.buildCluster();
        
        Integer [] COUNT = new Integer[max_rand];
 

Modified: incubator/pig/trunk/test/org/apache/pig/test/TestMapReduce.java
URL: 
http://svn.apache.org/viewvc/incubator/pig/trunk/test/org/apache/pig/test/TestMapReduce.java?rev=630355&r1=630354&r2=630355&view=diff
==============================================================================
--- incubator/pig/trunk/test/org/apache/pig/test/TestMapReduce.java (original)
+++ incubator/pig/trunk/test/org/apache/pig/test/TestMapReduce.java Fri Feb 22 
15:39:11 2008
@@ -48,6 +48,7 @@
 public class TestMapReduce extends TestCase {
 
        private String initString = "mapreduce";
+       MiniCluster cluster = MiniCluster.buildCluster();
 
        @Test
     public void testBigGroupAll() throws Throwable {

Modified: incubator/pig/trunk/test/org/apache/pig/test/TestOrderBy.java
URL: 
http://svn.apache.org/viewvc/incubator/pig/trunk/test/org/apache/pig/test/TestOrderBy.java?rev=630355&r1=630354&r2=630355&view=diff
==============================================================================
--- incubator/pig/trunk/test/org/apache/pig/test/TestOrderBy.java (original)
+++ incubator/pig/trunk/test/org/apache/pig/test/TestOrderBy.java Fri Feb 22 
15:39:11 2008
@@ -34,6 +34,7 @@
     private String initString = "mapreduce";
     private static final int DATALEN = 1024;
     private String[][] DATA = new String[2][DATALEN];
+    MiniCluster cluster = MiniCluster.buildCluster();
     
     private PigServer pig;
     private File tmpFile;

Modified: incubator/pig/trunk/test/org/apache/pig/test/TestPi.java
URL: 
http://svn.apache.org/viewvc/incubator/pig/trunk/test/org/apache/pig/test/TestPi.java?rev=630355&r1=630354&r2=630355&view=diff
==============================================================================
--- incubator/pig/trunk/test/org/apache/pig/test/TestPi.java (original)
+++ incubator/pig/trunk/test/org/apache/pig/test/TestPi.java Fri Feb 22 
15:39:11 2008
@@ -53,6 +53,7 @@
        private long total = ((defaultBlockSize >> 20) / 10) << 20;
        private int inCircle = 0;
        private String initString = "mapreduce";
+       MiniCluster cluster = MiniCluster.buildCluster();
 
        private long totalLength = 0, totalLengthTest = 0;
 

Modified: incubator/pig/trunk/test/org/apache/pig/test/TestPigServer.java
URL: 
http://svn.apache.org/viewvc/incubator/pig/trunk/test/org/apache/pig/test/TestPigServer.java?rev=630355&r1=630354&r2=630355&view=diff
==============================================================================
--- incubator/pig/trunk/test/org/apache/pig/test/TestPigServer.java (original)
+++ incubator/pig/trunk/test/org/apache/pig/test/TestPigServer.java Fri Feb 22 
15:39:11 2008
@@ -19,6 +19,7 @@
 
 public class TestPigServer extends TestCase {
     private PigServer pig = null;
+    MiniCluster cluster = MiniCluster.buildCluster();
     
     private void initPigServer() throws Throwable {
         if (pig == null) {

Modified: incubator/pig/trunk/test/org/apache/pig/test/TestPigSplit.java
URL: 
http://svn.apache.org/viewvc/incubator/pig/trunk/test/org/apache/pig/test/TestPigSplit.java?rev=630355&r1=630354&r2=630355&view=diff
==============================================================================
--- incubator/pig/trunk/test/org/apache/pig/test/TestPigSplit.java (original)
+++ incubator/pig/trunk/test/org/apache/pig/test/TestPigSplit.java Fri Feb 22 
15:39:11 2008
@@ -35,6 +35,7 @@
 
 public class TestPigSplit extends TestCase {
        PigServer pig;
+       MiniCluster cluster = MiniCluster.buildCluster();
        
        @Override
        protected void setUp() throws Exception {

Modified: incubator/pig/trunk/test/org/apache/pig/test/TestStore.java
URL: 
http://svn.apache.org/viewvc/incubator/pig/trunk/test/org/apache/pig/test/TestStore.java?rev=630355&r1=630354&r2=630355&view=diff
==============================================================================
--- incubator/pig/trunk/test/org/apache/pig/test/TestStore.java (original)
+++ incubator/pig/trunk/test/org/apache/pig/test/TestStore.java Fri Feb 22 
15:39:11 2008
@@ -33,6 +33,7 @@
 
        private String initString = "mapreduce";
        private int LOOP_COUNT = 1024;
+       MiniCluster cluster = MiniCluster.buildCluster();
        
        String fileName;
        String tmpFile1, tmpFile2;


Reply via email to