Author: daijy
Date: Tue Mar 19 23:42:58 2013
New Revision: 1458574
URL: http://svn.apache.org/r1458574
Log:
HCATALOG-624: Fix HCat for hadoop 2 & HCATALOG-637: Upgrade Pig from 0.8 to 0.10
Added:
incubator/hcatalog/trunk/shims/src/20/java/org/apache/hadoop/
incubator/hcatalog/trunk/shims/src/20/java/org/apache/hadoop/mapred/
incubator/hcatalog/trunk/shims/src/20/java/org/apache/hadoop/mapred/TempletonJobTracker.java
incubator/hcatalog/trunk/shims/src/23/java/org/apache/hadoop/
incubator/hcatalog/trunk/shims/src/23/java/org/apache/hadoop/mapred/
incubator/hcatalog/trunk/shims/src/23/java/org/apache/hadoop/mapred/TempletonJobTracker.java
Removed:
incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hadoop/mapred/TempletonJobTracker.java
Modified:
incubator/hcatalog/trunk/build.xml
incubator/hcatalog/trunk/core/pom.xml
incubator/hcatalog/trunk/core/src/main/java/org/apache/hcatalog/common/HCatUtil.java
incubator/hcatalog/trunk/core/src/main/java/org/apache/hcatalog/mapreduce/MultiOutputFormat.java
incubator/hcatalog/trunk/core/src/test/java/org/apache/hcatalog/HcatTestUtils.java
incubator/hcatalog/trunk/core/src/test/java/org/apache/hcatalog/mapreduce/HCatMapReduceTest.java
incubator/hcatalog/trunk/core/src/test/java/org/apache/hcatalog/mapreduce/TestHCatDynamicPartitioned.java
incubator/hcatalog/trunk/core/src/test/java/org/apache/hcatalog/mapreduce/TestHCatMultiOutputFormat.java
incubator/hcatalog/trunk/core/src/test/java/org/apache/hcatalog/mapreduce/TestHCatPartitionPublish.java
incubator/hcatalog/trunk/core/src/test/java/org/apache/hcatalog/mapreduce/TestMultiOutputFormat.java
incubator/hcatalog/trunk/core/src/test/java/org/apache/hcatalog/mapreduce/TestSequenceFileReadWrite.java
incubator/hcatalog/trunk/hcatalog-pig-adapter/pom.xml
incubator/hcatalog/trunk/hcatalog-pig-adapter/src/test/java/org/apache/hcatalog/pig/TestHCatLoaderComplexSchema.java
incubator/hcatalog/trunk/hcatalog-pig-adapter/src/test/java/org/apache/hcatalog/pig/TestPigHCatUtil.java
incubator/hcatalog/trunk/pom.xml
incubator/hcatalog/trunk/storage-handlers/hbase/pom.xml
incubator/hcatalog/trunk/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/HBaseBulkOutputFormat.java
incubator/hcatalog/trunk/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/ManyMiniCluster.java
incubator/hcatalog/trunk/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/SkeletonHBaseTest.java
incubator/hcatalog/trunk/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestHBaseBulkOutputFormat.java
incubator/hcatalog/trunk/webhcat/svr/pom.xml
incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/DeleteDelegator.java
incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/ListDelegator.java
incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/StatusDelegator.java
incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/TempletonDelegator.java
Modified: incubator/hcatalog/trunk/build.xml
URL:
http://svn.apache.org/viewvc/incubator/hcatalog/trunk/build.xml?rev=1458574&r1=1458573&r2=1458574&view=diff
==============================================================================
--- incubator/hcatalog/trunk/build.xml (original)
+++ incubator/hcatalog/trunk/build.xml Tue Mar 19 23:42:58 2013
@@ -69,13 +69,13 @@
<property name="_mvn.hadoop.profile" value="${mvn.hadoop.profile}"/>
</ant>
<ant target="jar" dir="webhcat/svr" inheritAll="false">
- <property name="_mvn.hadoop.profile" value="hadoop20"/>
+ <property name="_mvn.hadoop.profile" value="${mvn.hadoop.profile}"/>
</ant>
<ant target="jar" dir="webhcat/java-client" inheritAll="false">
<property name="_mvn.hadoop.profile" value="${mvn.hadoop.profile}"/>
</ant>
<ant target="jar" dir="storage-handlers/hbase" inheritAll="false">
- <property name="_mvn.hadoop.profile" value="hadoop20"/>
+ <property name="_mvn.hadoop.profile" value="${mvn.hadoop.profile}"/>
</ant>
</target>
@@ -192,7 +192,7 @@
<packageset dir="webhcat/svr/src/main/java"/>
<packageset dir="webhcat/java-client/src/main/java"/>
<classpath>
- <fileset dir="core/build/lib/compile"/>
+ <fileset dir="core/build/lib/test"/>
<fileset dir="hcatalog-pig-adapter/build/lib/compile"/>
<fileset dir="hcatalog-pig-adapter/build/lib/provided"/>
<fileset dir="server-extensions/build/lib/compile"/>
Modified: incubator/hcatalog/trunk/core/pom.xml
URL:
http://svn.apache.org/viewvc/incubator/hcatalog/trunk/core/pom.xml?rev=1458574&r1=1458573&r2=1458574&view=diff
==============================================================================
--- incubator/hcatalog/trunk/core/pom.xml (original)
+++ incubator/hcatalog/trunk/core/pom.xml Tue Mar 19 23:42:58 2013
@@ -89,13 +89,5 @@
<version>${hive.version}</version>
<scope>compile</scope>
</dependency>
-
- <!-- test scope -->
- <dependency>
- <groupId>org.apache.pig</groupId>
- <artifactId>pig</artifactId>
- <version>${pig.version}</version>
- <scope>test</scope>
- </dependency>
</dependencies>
</project>
Modified:
incubator/hcatalog/trunk/core/src/main/java/org/apache/hcatalog/common/HCatUtil.java
URL:
http://svn.apache.org/viewvc/incubator/hcatalog/trunk/core/src/main/java/org/apache/hcatalog/common/HCatUtil.java?rev=1458574&r1=1458573&r2=1458574&view=diff
==============================================================================
---
incubator/hcatalog/trunk/core/src/main/java/org/apache/hcatalog/common/HCatUtil.java
(original)
+++
incubator/hcatalog/trunk/core/src/main/java/org/apache/hcatalog/common/HCatUtil.java
Tue Mar 19 23:42:58 2013
@@ -615,4 +615,12 @@ public class HCatUtil {
jobConf.set(entry.getKey(), entry.getValue());
}
}
+
+
+ public static boolean isHadoop23() {
+ String version = org.apache.hadoop.util.VersionInfo.getVersion();
+ if (version.matches("\\b0\\.23\\..+\\b")||version.matches("\\b2\\..*"))
+ return true;
+ return false;
+ }
}
Modified:
incubator/hcatalog/trunk/core/src/main/java/org/apache/hcatalog/mapreduce/MultiOutputFormat.java
URL:
http://svn.apache.org/viewvc/incubator/hcatalog/trunk/core/src/main/java/org/apache/hcatalog/mapreduce/MultiOutputFormat.java?rev=1458574&r1=1458573&r2=1458574&view=diff
==============================================================================
---
incubator/hcatalog/trunk/core/src/main/java/org/apache/hcatalog/mapreduce/MultiOutputFormat.java
(original)
+++
incubator/hcatalog/trunk/core/src/main/java/org/apache/hcatalog/mapreduce/MultiOutputFormat.java
Tue Mar 19 23:42:58 2013
@@ -47,6 +47,7 @@ import org.apache.hadoop.mapreduce.Recor
import org.apache.hadoop.mapreduce.TaskAttemptContext;
import org.apache.hadoop.mapreduce.TaskInputOutputContext;
import org.apache.hadoop.util.ReflectionUtils;
+import org.apache.hcatalog.common.HCatUtil;
import org.apache.hcatalog.shims.HCatHadoopShims;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -152,8 +153,14 @@ public class MultiOutputFormat extends O
configsToMerge.put("tmparchives", COMMA_DELIM);
configsToMerge.put(HCatHadoopShims.Instance.get().getPropertyName(HCatHadoopShims.PropertyName.CACHE_ARCHIVES),
COMMA_DELIM);
configsToMerge.put(HCatHadoopShims.Instance.get().getPropertyName(HCatHadoopShims.PropertyName.CACHE_FILES),
COMMA_DELIM);
- configsToMerge.put("mapred.job.classpath.archives",
System.getProperty("path.separator"));
- configsToMerge.put("mapred.job.classpath.files",
System.getProperty("path.separator"));
+ String fileSep;
+ if (HCatUtil.isHadoop23()) {
+ fileSep = ",";
+ } else {
+ fileSep = System.getProperty("path.separator");
+ }
+ configsToMerge.put("mapred.job.classpath.archives", fileSep);
+ configsToMerge.put("mapred.job.classpath.files", fileSep);
}
/**
Modified:
incubator/hcatalog/trunk/core/src/test/java/org/apache/hcatalog/HcatTestUtils.java
URL:
http://svn.apache.org/viewvc/incubator/hcatalog/trunk/core/src/test/java/org/apache/hcatalog/HcatTestUtils.java?rev=1458574&r1=1458573&r2=1458574&view=diff
==============================================================================
---
incubator/hcatalog/trunk/core/src/test/java/org/apache/hcatalog/HcatTestUtils.java
(original)
+++
incubator/hcatalog/trunk/core/src/test/java/org/apache/hcatalog/HcatTestUtils.java
Tue Mar 19 23:42:58 2013
@@ -97,11 +97,4 @@ public class HcatTestUtils {
}
}
-
- public static boolean isHadoop23() {
- String version = org.apache.hadoop.util.VersionInfo.getVersion();
- if (version.matches("\\b0\\.23\\..+\\b"))
- return true;
- return false;
- }
}
Modified:
incubator/hcatalog/trunk/core/src/test/java/org/apache/hcatalog/mapreduce/HCatMapReduceTest.java
URL:
http://svn.apache.org/viewvc/incubator/hcatalog/trunk/core/src/test/java/org/apache/hcatalog/mapreduce/HCatMapReduceTest.java?rev=1458574&r1=1458573&r2=1458574&view=diff
==============================================================================
---
incubator/hcatalog/trunk/core/src/test/java/org/apache/hcatalog/mapreduce/HCatMapReduceTest.java
(original)
+++
incubator/hcatalog/trunk/core/src/test/java/org/apache/hcatalog/mapreduce/HCatMapReduceTest.java
Tue Mar 19 23:42:58 2013
@@ -51,7 +51,6 @@ import org.apache.hadoop.mapreduce.JobSt
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
-import org.apache.hcatalog.HcatTestUtils;
import org.apache.hcatalog.common.HCatConstants;
import org.apache.hcatalog.common.HCatUtil;
import org.apache.hcatalog.data.DefaultHCatRecord;
@@ -281,7 +280,7 @@ public abstract class HCatMapReduceTest
.findCounter("FILE_BYTES_READ").getValue() > 0);
}
- if (!HcatTestUtils.isHadoop23()) {
+ if (!HCatUtil.isHadoop23()) {
// Local mode outputcommitter hook is not invoked in Hadoop 1.x
if (success) {
new FileOutputCommitterContainer(job, null).commitJob(job);
Modified:
incubator/hcatalog/trunk/core/src/test/java/org/apache/hcatalog/mapreduce/TestHCatDynamicPartitioned.java
URL:
http://svn.apache.org/viewvc/incubator/hcatalog/trunk/core/src/test/java/org/apache/hcatalog/mapreduce/TestHCatDynamicPartitioned.java?rev=1458574&r1=1458573&r2=1458574&view=diff
==============================================================================
---
incubator/hcatalog/trunk/core/src/test/java/org/apache/hcatalog/mapreduce/TestHCatDynamicPartitioned.java
(original)
+++
incubator/hcatalog/trunk/core/src/test/java/org/apache/hcatalog/mapreduce/TestHCatDynamicPartitioned.java
Tue Mar 19 23:42:58 2013
@@ -23,11 +23,13 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
+import junit.framework.Assert;
+
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.serde.serdeConstants;
import org.apache.hadoop.mapreduce.Job;
-import org.apache.hcatalog.HcatTestUtils;
+import org.apache.hcatalog.common.HCatUtil;
import org.apache.hcatalog.common.ErrorType;
import org.apache.hcatalog.common.HCatConstants;
import org.apache.hcatalog.common.HCatException;
@@ -141,20 +143,23 @@ public class TestHCatDynamicPartitioned
try {
generateWriteRecords(NUM_RECORDS, NUM_PARTITIONS, 0);
Job job = runMRCreate(null, dataColumns, writeRecords,
NUM_RECORDS, false);
- if (HcatTestUtils.isHadoop23()) {
- new FileOutputCommitterContainer(job, null).cleanupJob(job);
+
+ if (HCatUtil.isHadoop23()) {
+ Assert.assertTrue(job.isSuccessful()==false);
}
} catch (IOException e) {
exc = e;
}
- assertTrue(exc != null);
- assertTrue(exc instanceof HCatException);
- assertTrue("Got exception of type [" + ((HCatException)
exc).getErrorType().toString()
- + "] Expected ERROR_PUBLISHING_PARTITION or ERROR_MOVE_FAILED",
- (ErrorType.ERROR_PUBLISHING_PARTITION == ((HCatException)
exc).getErrorType())
- || (ErrorType.ERROR_MOVE_FAILED == ((HCatException)
exc).getErrorType())
- );
+ if (!HCatUtil.isHadoop23()) {
+ assertTrue(exc != null);
+ assertTrue(exc instanceof HCatException);
+ assertTrue("Got exception of type [" + ((HCatException)
exc).getErrorType().toString()
+ + "] Expected ERROR_PUBLISHING_PARTITION or
ERROR_MOVE_FAILED",
+ (ErrorType.ERROR_PUBLISHING_PARTITION == ((HCatException)
exc).getErrorType())
+ || (ErrorType.ERROR_MOVE_FAILED ==
((HCatException) exc).getErrorType())
+ );
+ }
}
//TODO 1.0 miniCluster is slow this test times out, make it work
Modified:
incubator/hcatalog/trunk/core/src/test/java/org/apache/hcatalog/mapreduce/TestHCatMultiOutputFormat.java
URL:
http://svn.apache.org/viewvc/incubator/hcatalog/trunk/core/src/test/java/org/apache/hcatalog/mapreduce/TestHCatMultiOutputFormat.java?rev=1458574&r1=1458573&r2=1458574&view=diff
==============================================================================
---
incubator/hcatalog/trunk/core/src/test/java/org/apache/hcatalog/mapreduce/TestHCatMultiOutputFormat.java
(original)
+++
incubator/hcatalog/trunk/core/src/test/java/org/apache/hcatalog/mapreduce/TestHCatMultiOutputFormat.java
Tue Mar 19 23:42:58 2013
@@ -175,6 +175,9 @@ public class TestHCatMultiOutputFormat {
// LocalJobRunner does not work with mapreduce OutputCommitter. So need
// to use MiniMRCluster. MAPREDUCE-2350
Configuration conf = new Configuration(true);
+ conf.set("yarn.scheduler.capacity.root.queues", "default");
+ conf.set("yarn.scheduler.capacity.root.default.capacity", "100");
+
FileSystem fs = FileSystem.get(conf);
System.setProperty("hadoop.log.dir", new File(workDir,
"/logs").getAbsolutePath());
mrCluster = new MiniMRCluster(1, fs.getUri().toString(), 1, null, null,
Modified:
incubator/hcatalog/trunk/core/src/test/java/org/apache/hcatalog/mapreduce/TestHCatPartitionPublish.java
URL:
http://svn.apache.org/viewvc/incubator/hcatalog/trunk/core/src/test/java/org/apache/hcatalog/mapreduce/TestHCatPartitionPublish.java?rev=1458574&r1=1458573&r2=1458574&view=diff
==============================================================================
---
incubator/hcatalog/trunk/core/src/test/java/org/apache/hcatalog/mapreduce/TestHCatPartitionPublish.java
(original)
+++
incubator/hcatalog/trunk/core/src/test/java/org/apache/hcatalog/mapreduce/TestHCatPartitionPublish.java
Tue Mar 19 23:42:58 2013
@@ -75,6 +75,9 @@ public class TestHCatPartitionPublish {
@BeforeClass
public static void setup() throws Exception {
Configuration conf = new Configuration(true);
+ conf.set("yarn.scheduler.capacity.root.queues", "default");
+ conf.set("yarn.scheduler.capacity.root.default.capacity", "100");
+
fs = FileSystem.get(conf);
System.setProperty("hadoop.log.dir", new File(fs.getWorkingDirectory()
.toString(), "/logs").getAbsolutePath());
Modified:
incubator/hcatalog/trunk/core/src/test/java/org/apache/hcatalog/mapreduce/TestMultiOutputFormat.java
URL:
http://svn.apache.org/viewvc/incubator/hcatalog/trunk/core/src/test/java/org/apache/hcatalog/mapreduce/TestMultiOutputFormat.java?rev=1458574&r1=1458573&r2=1458574&view=diff
==============================================================================
---
incubator/hcatalog/trunk/core/src/test/java/org/apache/hcatalog/mapreduce/TestMultiOutputFormat.java
(original)
+++
incubator/hcatalog/trunk/core/src/test/java/org/apache/hcatalog/mapreduce/TestMultiOutputFormat.java
Tue Mar 19 23:42:58 2013
@@ -23,6 +23,8 @@ import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.net.URI;
+import java.util.Arrays;
+import java.util.List;
import java.util.Random;
import java.util.StringTokenizer;
@@ -70,6 +72,9 @@ public class TestMultiOutputFormat {
public static void setup() throws IOException {
createWorkDir();
Configuration conf = new Configuration(true);
+ conf.set("yarn.scheduler.capacity.root.queues", "default");
+ conf.set("yarn.scheduler.capacity.root.default.capacity", "100");
+
fs = FileSystem.get(conf);
System.setProperty("hadoop.log.dir", new File(workDir,
"/logs").getAbsolutePath());
// LocalJobRunner does not work with mapreduce OutputCommitter. So need
@@ -134,12 +139,14 @@ public class TestMultiOutputFormat {
// Verify if the configs are merged
Path[] fileClassPaths =
DistributedCache.getFileClassPaths(job.getConfiguration());
- Assert.assertArrayEquals(new Path[]{new Path(inputFile), new
Path(dummyFile)},
- fileClassPaths);
- URI[] expectedCacheFiles = new URI[]{new
Path(inputFile).makeQualified(fs).toUri(),
- new Path(dummyFile).makeQualified(fs).toUri()};
+ List<Path> fileClassPathsList = Arrays.asList(fileClassPaths);
+ Assert.assertTrue(fileClassPathsList.contains(new Path(inputFile)));
+ Assert.assertTrue(fileClassPathsList.contains(new Path(dummyFile)));
+
URI[] cacheFiles =
DistributedCache.getCacheFiles(job.getConfiguration());
- Assert.assertArrayEquals(expectedCacheFiles, cacheFiles);
+ List<URI> cacheFilesList = Arrays.asList(cacheFiles);
+ Assert.assertTrue(cacheFilesList.contains(new
Path(inputFile).makeQualified(fs).toUri()));
+ Assert.assertTrue(cacheFilesList.contains(new
Path(dummyFile).makeQualified(fs).toUri()));
Assert.assertTrue(job.waitForCompletion(true));
Modified:
incubator/hcatalog/trunk/core/src/test/java/org/apache/hcatalog/mapreduce/TestSequenceFileReadWrite.java
URL:
http://svn.apache.org/viewvc/incubator/hcatalog/trunk/core/src/test/java/org/apache/hcatalog/mapreduce/TestSequenceFileReadWrite.java?rev=1458574&r1=1458573&r2=1458574&view=diff
==============================================================================
---
incubator/hcatalog/trunk/core/src/test/java/org/apache/hcatalog/mapreduce/TestSequenceFileReadWrite.java
(original)
+++
incubator/hcatalog/trunk/core/src/test/java/org/apache/hcatalog/mapreduce/TestSequenceFileReadWrite.java
Tue Mar 19 23:42:58 2013
@@ -170,7 +170,7 @@ public class TestSequenceFileReadWrite e
HCatOutputFormat.setSchema(job, getSchema());
job.setNumReduceTasks(0);
assertTrue(job.waitForCompletion(true));
- if (!HcatTestUtils.isHadoop23()) {
+ if (!HCatUtil.isHadoop23()) {
new FileOutputCommitterContainer(job, null).commitJob(job);
}
assertTrue(job.isSuccessful());
@@ -217,7 +217,7 @@ public class TestSequenceFileReadWrite e
job.setOutputFormatClass(HCatOutputFormat.class);
HCatOutputFormat.setSchema(job, getSchema());
assertTrue(job.waitForCompletion(true));
- if (!HcatTestUtils.isHadoop23()) {
+ if (!HCatUtil.isHadoop23()) {
new FileOutputCommitterContainer(job, null).commitJob(job);
}
assertTrue(job.isSuccessful());
Modified: incubator/hcatalog/trunk/hcatalog-pig-adapter/pom.xml
URL:
http://svn.apache.org/viewvc/incubator/hcatalog/trunk/hcatalog-pig-adapter/pom.xml?rev=1458574&r1=1458573&r2=1458574&view=diff
==============================================================================
--- incubator/hcatalog/trunk/hcatalog-pig-adapter/pom.xml (original)
+++ incubator/hcatalog/trunk/hcatalog-pig-adapter/pom.xml Tue Mar 19 23:42:58
2013
@@ -43,11 +43,5 @@
<version>${hcatalog.version}</version>
<scope>compile</scope>
</dependency>
- <dependency>
- <groupId>org.apache.pig</groupId>
- <artifactId>pig</artifactId>
- <version>${pig.version}</version>
- <scope>compile</scope>
- </dependency>
</dependencies>
</project>
Modified:
incubator/hcatalog/trunk/hcatalog-pig-adapter/src/test/java/org/apache/hcatalog/pig/TestHCatLoaderComplexSchema.java
URL:
http://svn.apache.org/viewvc/incubator/hcatalog/trunk/hcatalog-pig-adapter/src/test/java/org/apache/hcatalog/pig/TestHCatLoaderComplexSchema.java?rev=1458574&r1=1458573&r2=1458574&view=diff
==============================================================================
---
incubator/hcatalog/trunk/hcatalog-pig-adapter/src/test/java/org/apache/hcatalog/pig/TestHCatLoaderComplexSchema.java
(original)
+++
incubator/hcatalog/trunk/hcatalog-pig-adapter/src/test/java/org/apache/hcatalog/pig/TestHCatLoaderComplexSchema.java
Tue Mar 19 23:42:58 2013
@@ -112,7 +112,6 @@ public class TestHCatLoaderComplexSchema
@Test
public void testSyntheticComplexSchema() throws Exception {
String pigSchema =
- "(" +
"a: " +
"(" +
"aa: chararray, " +
@@ -124,8 +123,7 @@ public class TestHCatLoaderComplexSchema
")," +
"b: chararray, " +
"c: long, " +
- "d: { t: (da:long, db: ( dba: chararray, dbb: long), dc: { t:
(dca: long) } ) } " +
- ")";
+ "d: { t: (da:long, db: ( dba: chararray, dbb: long), dc: { t:
(dca: long) } ) } ";
// with extra structs
String tableSchema =
@@ -191,7 +189,7 @@ public class TestHCatLoaderComplexSchema
createTable(tablename, tableSchema);
PigServer server = new PigServer(ExecType.LOCAL);
server.setBatchOn();
- server.registerQuery("A = load '" + tablename + "Input' using
org.apache.hcatalog.pig.MockLoader() AS " + pigSchema + ";");
+ server.registerQuery("A = load '" + tablename + "Input' using
org.apache.hcatalog.pig.MockLoader() AS (" + pigSchema + ");");
Schema dumpedASchema = server.dumpSchema("A");
server.registerQuery("STORE A into '" + tablename + "' using
org.apache.hcatalog.pig.HCatStorer("
+ (provideSchemaToStorer ? "'', '" + pigSchema + "'" : "")
@@ -255,7 +253,7 @@ public class TestHCatLoaderComplexSchema
*/
@Test
public void testTupleInBagInTupleInBag() throws Exception {
- String pigSchema = "(a: { b : ( c: { d: (i : long) } ) })";
+ String pigSchema = "a: { b : ( c: { d: (i : long) } ) }";
String tableSchema = "a array< array< bigint > >";
@@ -279,7 +277,7 @@ public class TestHCatLoaderComplexSchema
@Test
public void testMapWithComplexData() throws Exception {
- String pigSchema = "(a: long, b: map[])";
+ String pigSchema = "a: long, b: map[]";
String tableSchema = "a bigint, b map<string, struct<aa:bigint,
ab:string>>";
List<Tuple> data = new ArrayList<Tuple>();
Modified:
incubator/hcatalog/trunk/hcatalog-pig-adapter/src/test/java/org/apache/hcatalog/pig/TestPigHCatUtil.java
URL:
http://svn.apache.org/viewvc/incubator/hcatalog/trunk/hcatalog-pig-adapter/src/test/java/org/apache/hcatalog/pig/TestPigHCatUtil.java?rev=1458574&r1=1458573&r2=1458574&view=diff
==============================================================================
---
incubator/hcatalog/trunk/hcatalog-pig-adapter/src/test/java/org/apache/hcatalog/pig/TestPigHCatUtil.java
(original)
+++
incubator/hcatalog/trunk/hcatalog-pig-adapter/src/test/java/org/apache/hcatalog/pig/TestPigHCatUtil.java
Tue Mar 19 23:42:58 2013
@@ -65,7 +65,7 @@ public class TestPigHCatUtil {
// When updating our pig dependency this will need updated.
System.setProperty(HCatConstants.HCAT_PIG_INNER_TUPLE_NAME, "t");
System.setProperty(HCatConstants.HCAT_PIG_INNER_FIELD_NAME,
"FIELDNAME_tuple");
- UDFContext.getUDFContext().setClientSystemProps();
+
UDFContext.getUDFContext().setClientSystemProps(System.getProperties());
// Define the expected schema.
ResourceFieldSchema[] bagSubFieldSchemas = new ResourceFieldSchema[1];
Modified: incubator/hcatalog/trunk/pom.xml
URL:
http://svn.apache.org/viewvc/incubator/hcatalog/trunk/pom.xml?rev=1458574&r1=1458573&r2=1458574&view=diff
==============================================================================
--- incubator/hcatalog/trunk/pom.xml (original)
+++ incubator/hcatalog/trunk/pom.xml Tue Mar 19 23:42:58 2013
@@ -27,18 +27,19 @@
<commons-io.version>2.4</commons-io.version>
<guava.version>11.0.2</guava.version>
<hadoop20.version>1.0.3</hadoop20.version>
- <hadoop23.version>0.23.3</hadoop23.version>
- <hbase.version>0.92.0</hbase.version>
+ <hadoop23.version>2.0.3-alpha</hadoop23.version>
+ <hbase.version>0.94.5</hbase.version>
<hcatalog.version>${project.version}</hcatalog.version>
- <hive.version>0.10.0</hive.version>
+ <hive.version>0.11.0-SNAPSHOT</hive.version>
<jackson.version>1.8.3</jackson.version>
<jersey.version>1.14</jersey.version>
<jetty.webhcat.version>7.6.0.v20120127</jetty.webhcat.version>
<jms.version>1.1</jms.version>
- <pig.version>0.8.0</pig.version>
+ <pig.version>0.10.1</pig.version>
<slf4j.version>1.6.1</slf4j.version>
<wadl-resourcedoc-doclet.version>1.4</wadl-resourcedoc-doclet.version>
<zookeeper.version>3.4.3</zookeeper.version>
+ <metrics-core.version>2.1.2</metrics-core.version>
</properties>
<parent>
@@ -83,6 +84,12 @@
<version>${hadoop20.version}</version>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.apache.pig</groupId>
+ <artifactId>pig</artifactId>
+ <version>${pig.version}</version>
+ <scope>compile</scope>
+ </dependency>
</dependencies>
</profile>
<profile>
@@ -108,10 +115,63 @@
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs</artifactId>
+ <version>${hadoop23.version}</version>
+ <classifier>tests</classifier>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-mapreduce-client-core</artifactId>
<version>${hadoop23.version}</version>
<scope>compile</scope>
</dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-jobclient</artifactId>
+ <version>${hadoop23.version}</version>
+ <classifier>tests</classifier>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-server-tests</artifactId>
+ <version>${hadoop23.version}</version>
+ <classifier>tests</classifier>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-app</artifactId>
+ <version>${hadoop23.version}</version>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <version>${hadoop23.version}</version>
+ <classifier>tests</classifier>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-hs</artifactId>
+ <version>${hadoop23.version}</version>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-minicluster</artifactId>
+ <version>${hadoop23.version}</version>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.pig</groupId>
+ <artifactId>pig</artifactId>
+ <version>${pig.version}</version>
+ <classifier>h2</classifier>
+ <scope>compile</scope>
+ </dependency>
</dependencies>
</profile>
</profiles>
Added:
incubator/hcatalog/trunk/shims/src/20/java/org/apache/hadoop/mapred/TempletonJobTracker.java
URL:
http://svn.apache.org/viewvc/incubator/hcatalog/trunk/shims/src/20/java/org/apache/hadoop/mapred/TempletonJobTracker.java?rev=1458574&view=auto
==============================================================================
---
incubator/hcatalog/trunk/shims/src/20/java/org/apache/hadoop/mapred/TempletonJobTracker.java
(added)
+++
incubator/hcatalog/trunk/shims/src/20/java/org/apache/hadoop/mapred/TempletonJobTracker.java
Tue Mar 19 23:42:58 2013
@@ -0,0 +1,95 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hadoop.mapred;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.ipc.RPC;
+import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.security.UserGroupInformation;
+
+/*
+ * Communicate with the JobTracker as a specific user.
+ */
+public class TempletonJobTracker {
+ private JobSubmissionProtocol cnx;
+
+ /**
+ * Create a connection to the Job Tracker.
+ */
+ public TempletonJobTracker(InetSocketAddress addr,
+ Configuration conf)
+ throws IOException {
+ UserGroupInformation ugi = UserGroupInformation.getLoginUser();
+ cnx = (JobSubmissionProtocol)
+ RPC.getProxy(JobSubmissionProtocol.class,
+ JobSubmissionProtocol.versionID,
+ addr,
+ ugi,
+ conf,
+ NetUtils.getSocketFactory(conf,
+ JobSubmissionProtocol.class));
+ }
+
+ /**
+ * Grab a handle to a job that is already known to the JobTracker.
+ *
+ * @return Profile of the job, or null if not found.
+ */
+ public JobProfile getJobProfile(JobID jobid)
+ throws IOException {
+ return cnx.getJobProfile(jobid);
+ }
+
+ /**
+ * Grab a handle to a job that is already known to the JobTracker.
+ *
+ * @return Status of the job, or null if not found.
+ */
+ public JobStatus getJobStatus(JobID jobid)
+ throws IOException {
+ return cnx.getJobStatus(jobid);
+ }
+
+
+ /**
+ * Kill a job.
+ */
+ public void killJob(JobID jobid)
+ throws IOException {
+ cnx.killJob(jobid);
+ }
+
+ /**
+ * Get all the jobs submitted.
+ */
+ public JobStatus[] getAllJobs()
+ throws IOException {
+ return cnx.getAllJobs();
+ }
+
+ /**
+ * Close the connection to the Job Tracker.
+ */
+ public void close() {
+ RPC.stopProxy(cnx);
+ }
+}
Added:
incubator/hcatalog/trunk/shims/src/23/java/org/apache/hadoop/mapred/TempletonJobTracker.java
URL:
http://svn.apache.org/viewvc/incubator/hcatalog/trunk/shims/src/23/java/org/apache/hadoop/mapred/TempletonJobTracker.java?rev=1458574&view=auto
==============================================================================
---
incubator/hcatalog/trunk/shims/src/23/java/org/apache/hadoop/mapred/TempletonJobTracker.java
(added)
+++
incubator/hcatalog/trunk/shims/src/23/java/org/apache/hadoop/mapred/TempletonJobTracker.java
Tue Mar 19 23:42:58 2013
@@ -0,0 +1,96 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hadoop.mapred;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.UserGroupInformation;
+
+/*
+ * Communicate with the JobTracker as a specific user.
+ */
+public class TempletonJobTracker {
+ private JobClient jc;
+
+ /**
+ * Create a connection to the Job Tracker.
+ */
+ public TempletonJobTracker(InetSocketAddress addr,
+ Configuration conf)
+ throws IOException {
+
+ jc = new JobClient(conf);
+ }
+
+ /**
+ * Grab a handle to a job that is already known to the JobTracker.
+ *
+ * @return Profile of the job, or null if not found.
+ */
+ public JobProfile getJobProfile(JobID jobid)
+ throws IOException {
+ RunningJob rj = jc.getJob(jobid);
+ JobStatus jobStatus = rj.getJobStatus();
+ JobProfile jobProfile = new JobProfile(jobStatus.getUsername(),
jobStatus.getJobID(),
+ jobStatus.getJobFile(), jobStatus.getTrackingUrl(),
jobStatus.getJobName());
+ return jobProfile;
+ }
+
+ /**
+ * Grab a handle to a job that is already known to the JobTracker.
+ *
+ * @return Status of the job, or null if not found.
+ */
+ public JobStatus getJobStatus(JobID jobid)
+ throws IOException {
+ RunningJob rj = jc.getJob(jobid);
+ JobStatus jobStatus = rj.getJobStatus();
+ return jobStatus;
+ }
+
+
+ /**
+ * Kill a job.
+ */
+ public void killJob(JobID jobid)
+ throws IOException {
+ RunningJob rj = jc.getJob(jobid);
+ rj.killJob();
+ }
+
+ /**
+ * Get all the jobs submitted.
+ */
+ public JobStatus[] getAllJobs()
+ throws IOException {
+ return jc.getAllJobs();
+ }
+
+ /**
+ * Close the connection to the Job Tracker.
+ */
+ public void close() {
+ try {
+ jc.close();
+ } catch (IOException e) {
+ }
+ }
+}
Modified: incubator/hcatalog/trunk/storage-handlers/hbase/pom.xml
URL:
http://svn.apache.org/viewvc/incubator/hcatalog/trunk/storage-handlers/hbase/pom.xml?rev=1458574&r1=1458573&r2=1458574&view=diff
==============================================================================
--- incubator/hcatalog/trunk/storage-handlers/hbase/pom.xml (original)
+++ incubator/hcatalog/trunk/storage-handlers/hbase/pom.xml Tue Mar 19 23:42:58
2013
@@ -61,6 +61,12 @@
<version>${zookeeper.version}</version>
<scope>compile</scope>
</dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase</artifactId>
+ <version>${hbase.version}</version>
+ <scope>compile</scope>
+ </dependency>
<!-- test scope -->
<dependency>
@@ -89,5 +95,11 @@
<classifier>tests</classifier>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>com.yammer.metrics</groupId>
+ <artifactId>metrics-core</artifactId>
+ <version>${metrics-core.version}</version>
+ <scope>test</scope>
+ </dependency>
</dependencies>
</project>
Modified:
incubator/hcatalog/trunk/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/HBaseBulkOutputFormat.java
URL:
http://svn.apache.org/viewvc/incubator/hcatalog/trunk/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/HBaseBulkOutputFormat.java?rev=1458574&r1=1458573&r2=1458574&view=diff
==============================================================================
---
incubator/hcatalog/trunk/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/HBaseBulkOutputFormat.java
(original)
+++
incubator/hcatalog/trunk/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/HBaseBulkOutputFormat.java
Tue Mar 19 23:42:58 2013
@@ -144,7 +144,7 @@ class HBaseBulkOutputFormat extends HBas
@Override
public void commitTask(TaskAttemptContext taskContext)
throws IOException {
- baseOutputCommitter.commitTask(taskContext);
+ // baseOutputCommitter.commitTask(taskContext);
}
@Override
Modified:
incubator/hcatalog/trunk/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/ManyMiniCluster.java
URL:
http://svn.apache.org/viewvc/incubator/hcatalog/trunk/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/ManyMiniCluster.java?rev=1458574&r1=1458573&r2=1458574&view=diff
==============================================================================
---
incubator/hcatalog/trunk/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/ManyMiniCluster.java
(original)
+++
incubator/hcatalog/trunk/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/ManyMiniCluster.java
Tue Mar 19 23:42:58 2013
@@ -210,6 +210,8 @@ public class ManyMiniCluster {
jobConf = new JobConf();
jobConf.setInt("mapred.submit.replication", 1);
+ jobConf.set("yarn.scheduler.capacity.root.queues", "default");
+ jobConf.set("yarn.scheduler.capacity.root.default.capacity",
"100");
//conf.set("hadoop.job.history.location",new
File(workDir).getAbsolutePath()+"/history");
System.setProperty("hadoop.log.dir", new File(workDir,
"/logs").getAbsolutePath());
Modified:
incubator/hcatalog/trunk/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/SkeletonHBaseTest.java
URL:
http://svn.apache.org/viewvc/incubator/hcatalog/trunk/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/SkeletonHBaseTest.java?rev=1458574&r1=1458573&r2=1458574&view=diff
==============================================================================
---
incubator/hcatalog/trunk/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/SkeletonHBaseTest.java
(original)
+++
incubator/hcatalog/trunk/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/SkeletonHBaseTest.java
Tue Mar 19 23:42:58 2013
@@ -30,6 +30,7 @@ import java.util.Set;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HTableDescriptor;
@@ -203,10 +204,12 @@ public abstract class SkeletonHBaseTest
} finally {
System.out.println("Trying to cleanup: " + testDir);
try {
- FileUtil.fullyDelete(new File(testDir));
+ FileSystem fs = FileSystem.get(jobConf);
+ fs.delete(new Path(testDir), true);
} catch (IOException e) {
throw new IllegalStateException("Failed to cleanup
test dir", e);
}
+
}
}
}
Modified:
incubator/hcatalog/trunk/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestHBaseBulkOutputFormat.java
URL:
http://svn.apache.org/viewvc/incubator/hcatalog/trunk/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestHBaseBulkOutputFormat.java?rev=1458574&r1=1458573&r2=1458574&view=diff
==============================================================================
---
incubator/hcatalog/trunk/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestHBaseBulkOutputFormat.java
(original)
+++
incubator/hcatalog/trunk/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestHBaseBulkOutputFormat.java
Tue Mar 19 23:42:58 2013
@@ -183,6 +183,8 @@ public class TestHBaseBulkOutputFormat e
//create table
conf.set(HBaseConstants.PROPERTY_OUTPUT_TABLE_NAME_KEY, tableName);
+ conf.set("yarn.scheduler.capacity.root.queues", "default");
+ conf.set("yarn.scheduler.capacity.root.default.capacity", "100");
createTable(tableName, new String[]{familyName});
String data[] = {"1,english:one,spanish:uno",
Modified: incubator/hcatalog/trunk/webhcat/svr/pom.xml
URL:
http://svn.apache.org/viewvc/incubator/hcatalog/trunk/webhcat/svr/pom.xml?rev=1458574&r1=1458573&r2=1458574&view=diff
==============================================================================
--- incubator/hcatalog/trunk/webhcat/svr/pom.xml (original)
+++ incubator/hcatalog/trunk/webhcat/svr/pom.xml Tue Mar 19 23:42:58 2013
@@ -48,12 +48,6 @@
<version>${hcatalog.version}</version>
<scope>provided</scope>
</dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-core</artifactId>
- <version>${hadoop20.version}</version>
- <scope>provided</scope>
- </dependency>
<!-- compile scope - this is packaged -->
<dependency>
Modified:
incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/DeleteDelegator.java
URL:
http://svn.apache.org/viewvc/incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/DeleteDelegator.java?rev=1458574&r1=1458573&r2=1458574&view=diff
==============================================================================
---
incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/DeleteDelegator.java
(original)
+++
incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/DeleteDelegator.java
Tue Mar 19 23:42:58 2013
@@ -20,7 +20,6 @@ package org.apache.hcatalog.templeton;
import java.io.IOException;
import org.apache.hadoop.mapred.JobID;
-import org.apache.hadoop.mapred.JobTracker;
import org.apache.hadoop.mapred.TempletonJobTracker;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hcatalog.templeton.tool.JobState;
@@ -40,7 +39,7 @@ public class DeleteDelegator extends Tem
TempletonJobTracker tracker = null;
JobState state = null;
try {
- tracker = new TempletonJobTracker(JobTracker.getAddress(appConf),
+ tracker = new TempletonJobTracker(getAddress(appConf),
appConf);
JobID jobid = StatusDelegator.StringToJobID(id);
if (jobid == null)
Modified:
incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/ListDelegator.java
URL:
http://svn.apache.org/viewvc/incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/ListDelegator.java?rev=1458574&r1=1458573&r2=1458574&view=diff
==============================================================================
---
incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/ListDelegator.java
(original)
+++
incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/ListDelegator.java
Tue Mar 19 23:42:58 2013
@@ -23,7 +23,6 @@ import java.util.List;
import java.util.ArrayList;
import org.apache.hadoop.mapred.JobStatus;
-import org.apache.hadoop.mapred.JobTracker;
import org.apache.hadoop.mapred.TempletonJobTracker;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hcatalog.templeton.tool.JobState;
@@ -42,7 +41,7 @@ public class ListDelegator extends Templ
UserGroupInformation ugi = UserGroupInformation.createRemoteUser(user);
TempletonJobTracker tracker = null;
try {
- tracker = new TempletonJobTracker(JobTracker.getAddress(appConf),
+ tracker = new TempletonJobTracker(getAddress(appConf),
appConf);
ArrayList<String> ids = new ArrayList<String>();
Modified:
incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/StatusDelegator.java
URL:
http://svn.apache.org/viewvc/incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/StatusDelegator.java?rev=1458574&r1=1458573&r2=1458574&view=diff
==============================================================================
---
incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/StatusDelegator.java
(original)
+++
incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/StatusDelegator.java
Tue Mar 19 23:42:58 2013
@@ -25,7 +25,6 @@ import org.apache.commons.logging.LogFac
import org.apache.hadoop.mapred.JobID;
import org.apache.hadoop.mapred.JobProfile;
import org.apache.hadoop.mapred.JobStatus;
-import org.apache.hadoop.mapred.JobTracker;
import org.apache.hadoop.mapred.TempletonJobTracker;
import org.apache.hcatalog.templeton.tool.JobState;
@@ -45,7 +44,7 @@ public class StatusDelegator extends Tem
TempletonJobTracker tracker = null;
JobState state = null;
try {
- tracker = new TempletonJobTracker(JobTracker.getAddress(appConf),
+ tracker = new TempletonJobTracker(getAddress(appConf),
appConf);
JobID jobid = StatusDelegator.StringToJobID(id);
if (jobid == null)
Modified:
incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/TempletonDelegator.java
URL:
http://svn.apache.org/viewvc/incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/TempletonDelegator.java?rev=1458574&r1=1458573&r2=1458574&view=diff
==============================================================================
---
incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/TempletonDelegator.java
(original)
+++
incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/TempletonDelegator.java
Tue Mar 19 23:42:58 2013
@@ -18,6 +18,11 @@
*/
package org.apache.hcatalog.templeton;
+import java.net.InetSocketAddress;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.net.NetUtils;
+
/**
* The helper class for all the Templeton delegator classes. A
* delegator will call the underlying Templeton service such as hcat
@@ -29,4 +34,10 @@ public class TempletonDelegator {
public TempletonDelegator(AppConfig appConf) {
this.appConf = appConf;
}
+
+ public static InetSocketAddress getAddress(Configuration conf) {
+ String jobTrackerStr =
+ conf.get("mapred.job.tracker", "localhost:8012");
+ return NetUtils.createSocketAddr(jobTrackerStr);
+ }
}
