Repository: gora Updated Branches: refs/heads/master 90d1cc0af -> 74d78cbe5
GORA-346 Create shim layer to support multiple hadoop versions Project: http://git-wip-us.apache.org/repos/asf/gora/repo Commit: http://git-wip-us.apache.org/repos/asf/gora/commit/b51cdeea Tree: http://git-wip-us.apache.org/repos/asf/gora/tree/b51cdeea Diff: http://git-wip-us.apache.org/repos/asf/gora/diff/b51cdeea Branch: refs/heads/master Commit: b51cdeeadc1e79474acd250e51a77645ad676ce6 Parents: 90d1cc0 Author: Lewis John McGibbney <[email protected]> Authored: Sun Sep 7 15:49:09 2014 -0700 Committer: Lewis John McGibbney <[email protected]> Committed: Sun Sep 7 15:49:09 2014 -0700 ---------------------------------------------------------------------- gora-accumulo/pom.xml | 4 +- gora-cassandra/pom.xml | 6 +- gora-core/pom.xml | 34 +++++-- .../gora/mapreduce/GoraMapReduceUtils.java | 10 +- .../org/apache/gora/util/ReflectionUtils.java | 3 +- gora-shims-distribution/pom.xml | 79 +++++++++++++++ gora-shims-hadoop/pom.xml | 38 +++++++ .../apache/gora/shims/hadoop/HadoopShim.java | 55 ++++++++++ .../gora/shims/hadoop/HadoopShimFactory.java | 100 +++++++++++++++++++ .../gora/shims/hadoop/HadoopShimTest.java | 37 +++++++ gora-shims-hadoop1/pom.xml | 49 +++++++++ .../apache/gora/shims/hadoop1/HadoopShim1.java | 50 ++++++++++ .../gora/shims/hadoop1/TestHadoopShim1.java | 18 ++++ gora-shims-hadoop2/pom.xml | 43 ++++++++ .../apache/gora/shims/hadoop2/HadoopShim2.java | 56 +++++++++++ .../gora/shims/hadoop2/TestHadoopShim2.java | 18 ++++ gora-tutorial/pom.xml | 7 +- pom.xml | 29 +++++- 18 files changed, 613 insertions(+), 23 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/gora/blob/b51cdeea/gora-accumulo/pom.xml ---------------------------------------------------------------------- diff --git a/gora-accumulo/pom.xml b/gora-accumulo/pom.xml index b588a7d..3613298 100644 --- a/gora-accumulo/pom.xml +++ b/gora-accumulo/pom.xml @@ -155,7 +155,9 @@ <dependency> <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-test</artifactId> + <artifactId>hadoop-common</artifactId> + <version>${hadoop-2.version}</version> + <classifier>tests</classifier> </dependency> <dependency> http://git-wip-us.apache.org/repos/asf/gora/blob/b51cdeea/gora-cassandra/pom.xml ---------------------------------------------------------------------- diff --git a/gora-cassandra/pom.xml b/gora-cassandra/pom.xml index 9852562..8e140be 100644 --- a/gora-cassandra/pom.xml +++ b/gora-cassandra/pom.xml @@ -183,8 +183,10 @@ </dependency> <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-test</artifactId> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-common</artifactId> + <version>${hadoop-2.version}</version> + <classifier>tests</classifier> </dependency> </dependencies> http://git-wip-us.apache.org/repos/asf/gora/blob/b51cdeea/gora-core/pom.xml ---------------------------------------------------------------------- diff --git a/gora-core/pom.xml b/gora-core/pom.xml index c6ef288..1aac38e 100644 --- a/gora-core/pom.xml +++ b/gora-core/pom.xml @@ -103,10 +103,32 @@ <dependencies> <!-- Hadoop Dependencies --> <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-core</artifactId> + <groupId>org.apache.gora</groupId> + <artifactId>gora-shims-distribution</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-common</artifactId> + <version>${hadoop-2.version}</version> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-common</artifactId> + <version>${hadoop-2.version}</version> + <classifier>tests</classifier> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-mapreduce-client-jobclient</artifactId> + <version>${hadoop-2.version}</version> + <classifier>tests</classifier> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-mapreduce-client-core</artifactId> + <version>${hadoop-2.version}</version> </dependency> - <dependency> <groupId>org.apache.cxf</groupId> <artifactId>cxf-rt-frontend-jaxrs</artifactId> @@ -156,12 +178,6 @@ <artifactId>slf4j-simple</artifactId> <scope>provided</scope> </dependency> - - <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-test</artifactId> - <scope>test</scope> - </dependency> </dependencies> http://git-wip-us.apache.org/repos/asf/gora/blob/b51cdeea/gora-core/src/main/java/org/apache/gora/mapreduce/GoraMapReduceUtils.java ---------------------------------------------------------------------- diff --git a/gora-core/src/main/java/org/apache/gora/mapreduce/GoraMapReduceUtils.java b/gora-core/src/main/java/org/apache/gora/mapreduce/GoraMapReduceUtils.java index ac90157..69545de 100644 --- a/gora-core/src/main/java/org/apache/gora/mapreduce/GoraMapReduceUtils.java +++ b/gora-core/src/main/java/org/apache/gora/mapreduce/GoraMapReduceUtils.java @@ -21,6 +21,8 @@ package org.apache.gora.mapreduce; import java.io.IOException; import java.util.List; +import org.apache.gora.shims.hadoop.HadoopShim; +import org.apache.gora.shims.hadoop.HadoopShimFactory; import org.apache.gora.util.StringUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; @@ -36,6 +38,8 @@ import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; */ public class GoraMapReduceUtils { + private static HadoopShim hadoopShim = HadoopShimFactory.INSTANCE().getHadoopShim(); + public static class HelperInputFormat<K,V> extends FileInputFormat<K, V> { @Override public RecordReader<K, V> createRecordReader(InputSplit arg0, @@ -75,11 +79,11 @@ public class GoraMapReduceUtils { throws IOException { if(inputPath != null) { - Job job = new Job(conf); + Job job = hadoopShim.createJob(conf); FileInputFormat.addInputPath(job, new Path(inputPath)); - return new JobContext(job.getConfiguration(), null); + return hadoopShim.createJobContext(job.getConfiguration()); } - return new JobContext(conf, null); + return hadoopShim.createJobContext(conf); } } http://git-wip-us.apache.org/repos/asf/gora/blob/b51cdeea/gora-core/src/main/java/org/apache/gora/util/ReflectionUtils.java ---------------------------------------------------------------------- diff --git a/gora-core/src/main/java/org/apache/gora/util/ReflectionUtils.java b/gora-core/src/main/java/org/apache/gora/util/ReflectionUtils.java index 00f94e4..a43dd64 100644 --- a/gora-core/src/main/java/org/apache/gora/util/ReflectionUtils.java +++ b/gora-core/src/main/java/org/apache/gora/util/ReflectionUtils.java @@ -20,8 +20,6 @@ package org.apache.gora.util; import java.lang.reflect.Constructor; import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.Method; - import org.apache.avro.specific.SpecificRecordBuilderBase; import org.apache.gora.persistency.Persistent; @@ -105,6 +103,7 @@ public class ReflectionUtils { return clazz.getField(fieldName).get(null); } + @SuppressWarnings("unchecked") public static <T extends Persistent> SpecificRecordBuilderBase<T> classBuilder(Class<T> clazz) throws SecurityException , NoSuchMethodException, IllegalArgumentException, IllegalAccessException, InvocationTargetException { return (SpecificRecordBuilderBase<T>) clazz.getMethod("newBuilder").invoke(null); http://git-wip-us.apache.org/repos/asf/gora/blob/b51cdeea/gora-shims-distribution/pom.xml ---------------------------------------------------------------------- diff --git a/gora-shims-distribution/pom.xml b/gora-shims-distribution/pom.xml new file mode 100644 index 0000000..acff69a --- /dev/null +++ b/gora-shims-distribution/pom.xml @@ -0,0 +1,79 @@ + <!-- Licensed to the Apache Software Foundation (ASF) under one or more contributor + license agreements. See the NOTICE file distributed with this work for additional + information regarding copyright ownership. The ASF licenses this file to + You under the Apache License, Version 2.0 (the "License"); you may not use + this file except in compliance with the License. You may obtain a copy of + the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required + by applicable law or agreed to in writing, software distributed under the + License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS + OF ANY KIND, either express or implied. See the License for the specific + language governing permissions and limitations under the License. --> +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <modelVersion>4.0.0</modelVersion> + <packaging>bundle</packaging> + <parent> + <groupId>org.apache.gora</groupId> + <artifactId>gora</artifactId> + <version>0.5-SNAPSHOT</version> + </parent> + <artifactId>gora-shims-distribution</artifactId> + + <name>Apache Gora :: Shims Distribution</name> + <licenses> + <license> + <name>The Apache Software License, Version 2.0</name> + <url>http://www.apache.org/licenses/LICENSE-2.0.txt</url> + </license> + </licenses> + + <organization> + <name>The Apache Software Foundation</name> + <url>http://www.apache.org/</url> + </organization> + + <build> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-shade-plugin</artifactId> + <version>2.3</version> + <executions> + <execution> + <phase>package</phase> + <goals> + <goal>shade</goal> + </goals> + <configuration> + <artifactSet> + <includes> + <include>${project.groupId}:gora-shims-hadoop</include> + <include>${project.groupId}:gora-shims-hadoop1</include> + <include>${project.groupId}:gora-shims-hadoop2</include> + </includes> + </artifactSet> + </configuration> + </execution> + </executions> + </plugin> + </plugins> + </build> + + <dependencies> + <dependency> + <groupId>org.apache.gora</groupId> + <artifactId>gora-shims-hadoop1</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>org.apache.gora</groupId> + <artifactId>gora-shims-hadoop2</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>org.apache.gora</groupId> + <artifactId>gora-shims-hadoop</artifactId> + <version>${project.version}</version> + </dependency> + </dependencies> +</project> http://git-wip-us.apache.org/repos/asf/gora/blob/b51cdeea/gora-shims-hadoop/pom.xml ---------------------------------------------------------------------- diff --git a/gora-shims-hadoop/pom.xml b/gora-shims-hadoop/pom.xml new file mode 100644 index 0000000..a9f10ad --- /dev/null +++ b/gora-shims-hadoop/pom.xml @@ -0,0 +1,38 @@ + <!-- Licensed to the Apache Software Foundation (ASF) under one or more contributor + license agreements. See the NOTICE file distributed with this work for additional + information regarding copyright ownership. The ASF licenses this file to + You under the Apache License, Version 2.0 (the "License"); you may not use + this file except in compliance with the License. You may obtain a copy of + the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required + by applicable law or agreed to in writing, software distributed under the + License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS + OF ANY KIND, either express or implied. See the License for the specific + language governing permissions and limitations under the License. --> +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <modelVersion>4.0.0</modelVersion> + + <parent> + <groupId>org.apache.gora</groupId> + <artifactId>gora</artifactId> + <version>0.5-SNAPSHOT</version> + </parent> + <artifactId>gora-shims-hadoop</artifactId> + <packaging>bundle</packaging> + + <name>Apache Gora :: Shims Hadoop</name> + + <dependencies> + <dependency> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-core</artifactId> + <version>${hadoop-1.version}</version> + <optional>true</optional> + </dependency> + </dependencies> +</project> http://git-wip-us.apache.org/repos/asf/gora/blob/b51cdeea/gora-shims-hadoop/src/main/java/org/apache/gora/shims/hadoop/HadoopShim.java ---------------------------------------------------------------------- diff --git a/gora-shims-hadoop/src/main/java/org/apache/gora/shims/hadoop/HadoopShim.java b/gora-shims-hadoop/src/main/java/org/apache/gora/shims/hadoop/HadoopShim.java new file mode 100644 index 0000000..78d746a --- /dev/null +++ b/gora-shims-hadoop/src/main/java/org/apache/gora/shims/hadoop/HadoopShim.java @@ -0,0 +1,55 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.gora.shims.hadoop; + +import java.io.IOException; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.mapreduce.Job; +import org.apache.hadoop.mapreduce.JobContext; + +/** + * Provides version independent utility methods for interacting with Hadoop. + */ +public abstract class HadoopShim { + + public HadoopShim() { + super(); + } + + /** + * Create a new Hadoop {@link Job}. + * + * @param configuration + * The job's configuration. + * @return A newly created instance backed by the provided configuration. + * @throws IOException + */ + public abstract Job createJob(Configuration configuration) + throws IOException; + + /** + * Create a new {@link JobContext} object. + * + * @param configuration + * The configuration for this job context. + * @return A newly created instance. + */ + public abstract JobContext createJobContext(Configuration configuration); + +} http://git-wip-us.apache.org/repos/asf/gora/blob/b51cdeea/gora-shims-hadoop/src/main/java/org/apache/gora/shims/hadoop/HadoopShimFactory.java ---------------------------------------------------------------------- diff --git a/gora-shims-hadoop/src/main/java/org/apache/gora/shims/hadoop/HadoopShimFactory.java b/gora-shims-hadoop/src/main/java/org/apache/gora/shims/hadoop/HadoopShimFactory.java new file mode 100644 index 0000000..161f62e --- /dev/null +++ b/gora-shims-hadoop/src/main/java/org/apache/gora/shims/hadoop/HadoopShimFactory.java @@ -0,0 +1,100 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.gora.shims.hadoop; + +import java.util.HashMap; +import java.util.Map; + +import org.apache.hadoop.util.VersionInfo; + +/** + * Factory to create {@link HadoopShim} objects. + */ +public class HadoopShimFactory { + + /** + * Hadoop shim version mapping. + */ + private static final Map<String, String> HADOOP_VERSION_TO_IMPL_MAP = new HashMap<String, String>(); + + static { + HADOOP_VERSION_TO_IMPL_MAP.put("1", + "org.apache.gora.shims.hadoop1.HadoopShim1"); + HADOOP_VERSION_TO_IMPL_MAP.put("2", + "org.apache.gora.shims.hadoop2.HadoopShim2"); + } + + // package private + static class Singleton { + public static final HadoopShimFactory INSTANCE = new HadoopShimFactory(); + }; + + /** + * Access the {@link HadoopShimFactory} singleton. + * + * @return the shared instance of {@link HadoopShimFactory}. + */ + public static HadoopShimFactory INSTANCE() { + return Singleton.INSTANCE; + } + + /** + * Get the Hadoop shim for the Hadoop version on the class path. In case it + * fails to obtain an appropriate shim (i.e. unsupported Hadoop version), it + * throws a {@link RuntimeException}. + * + * Note that this method is potentially costly. + * + * @return A newly created instance of a {@link HadoopShim}. + */ + public HadoopShim getHadoopShim() { + String version = getMajorVersion(); + String className = HADOOP_VERSION_TO_IMPL_MAP.get(version); + + try { + Class<?> class1 = Class.forName(className); + return HadoopShim.class.cast(class1.newInstance()); + } catch (Exception e) { + throw new RuntimeException( + "Could not load Hadoop shim for version " + version + + ", className=" + className, e); + } + } + + /** + * Get the Hadoop major version number. + * + * @return The major version number of Hadoop. + */ + public String getMajorVersion() { + String vers = VersionInfo.getVersion(); + + String[] parts = vers.split("\\."); + if (parts.length < 2) { + throw new RuntimeException("Unable to parse Hadoop version: " + + vers + " (expected X.Y.* format)"); + } + return parts[0]; + + } + + // package private + HadoopShimFactory() { + } + +} http://git-wip-us.apache.org/repos/asf/gora/blob/b51cdeea/gora-shims-hadoop/src/test/java/org/apache/gora/shims/hadoop/HadoopShimTest.java ---------------------------------------------------------------------- diff --git a/gora-shims-hadoop/src/test/java/org/apache/gora/shims/hadoop/HadoopShimTest.java b/gora-shims-hadoop/src/test/java/org/apache/gora/shims/hadoop/HadoopShimTest.java new file mode 100644 index 0000000..762d124 --- /dev/null +++ b/gora-shims-hadoop/src/test/java/org/apache/gora/shims/hadoop/HadoopShimTest.java @@ -0,0 +1,37 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.gora.shims.hadoop; + +import org.junit.Test; +import static org.junit.Assert.*; + +public class HadoopShimTest { + + @Test + public void testGetMajorVersion() { + String version = HadoopShimFactory.INSTANCE().getMajorVersion(); + assertNotNull(version); + assertTrue(version.length() > 0); + } + + @Test + public void testInstance() { + assertNotNull(HadoopShimFactory.INSTANCE()); + } + +} http://git-wip-us.apache.org/repos/asf/gora/blob/b51cdeea/gora-shims-hadoop1/pom.xml ---------------------------------------------------------------------- diff --git a/gora-shims-hadoop1/pom.xml b/gora-shims-hadoop1/pom.xml new file mode 100644 index 0000000..c798f18 --- /dev/null +++ b/gora-shims-hadoop1/pom.xml @@ -0,0 +1,49 @@ + <!-- Licensed to the Apache Software Foundation (ASF) under one or more contributor + license agreements. See the NOTICE file distributed with this work for additional + information regarding copyright ownership. The ASF licenses this file to + You under the Apache License, Version 2.0 (the "License"); you may not use + this file except in compliance with the License. You may obtain a copy of + the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required + by applicable law or agreed to in writing, software distributed under the + License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS + OF ANY KIND, either express or implied. See the License for the specific + language governing permissions and limitations under the License. --> +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <modelVersion>4.0.0</modelVersion> + + <parent> + <groupId>org.apache.gora</groupId> + <artifactId>gora</artifactId> + <version>0.5-SNAPSHOT</version> + </parent> + <artifactId>gora-shims-hadoop1</artifactId> + <packaging>bundle</packaging> + + <name>Apache Gora :: Shims Hadoop 1.x</name> + + <dependencies> + <dependency> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-core</artifactId> + <version>${hadoop-1.version}</version> + <optional>true</optional> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-test</artifactId> + <version>${hadoop-1.version}</version> + <optional>true</optional> + </dependency> + <dependency> + <groupId>org.apache.gora</groupId> + <artifactId>gora-shims-hadoop</artifactId> + <version>${project.version}</version> + </dependency> + </dependencies> +</project> http://git-wip-us.apache.org/repos/asf/gora/blob/b51cdeea/gora-shims-hadoop1/src/main/java/org/apache/gora/shims/hadoop1/HadoopShim1.java ---------------------------------------------------------------------- diff --git a/gora-shims-hadoop1/src/main/java/org/apache/gora/shims/hadoop1/HadoopShim1.java b/gora-shims-hadoop1/src/main/java/org/apache/gora/shims/hadoop1/HadoopShim1.java new file mode 100644 index 0000000..9728af6 --- /dev/null +++ b/gora-shims-hadoop1/src/main/java/org/apache/gora/shims/hadoop1/HadoopShim1.java @@ -0,0 +1,50 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.gora.shims.hadoop1; + +import java.io.IOException; + +import org.apache.gora.shims.hadoop.HadoopShim; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.mapreduce.Job; +import org.apache.hadoop.mapreduce.JobContext; + +/** + * Provides shim implementation for Hadoop 1.x. + */ +public class HadoopShim1 extends HadoopShim { + + public HadoopShim1() { + super(); + } + + /** + * {@inheritDoc} + */ + public Job createJob(Configuration configuration) throws IOException { + return new Job(configuration); + } + + /** + * {@inheritDoc} + */ + public JobContext createJobContext(Configuration configuration) { + return new JobContext(configuration, null); + } + +} http://git-wip-us.apache.org/repos/asf/gora/blob/b51cdeea/gora-shims-hadoop1/src/test/java/org/apache/gora/shims/hadoop1/TestHadoopShim1.java ---------------------------------------------------------------------- diff --git a/gora-shims-hadoop1/src/test/java/org/apache/gora/shims/hadoop1/TestHadoopShim1.java b/gora-shims-hadoop1/src/test/java/org/apache/gora/shims/hadoop1/TestHadoopShim1.java new file mode 100644 index 0000000..f0f1533 --- /dev/null +++ b/gora-shims-hadoop1/src/test/java/org/apache/gora/shims/hadoop1/TestHadoopShim1.java @@ -0,0 +1,18 @@ +package org.apache.gora.shims.hadoop1; + +import org.apache.gora.shims.hadoop.HadoopShim; +import org.apache.gora.shims.hadoop.HadoopShimFactory; +import org.junit.Test; + +import static org.junit.Assert.*; + +public class TestHadoopShim1 { + + @Test + public void testCorrectVersion() { + HadoopShim shim = HadoopShimFactory.INSTANCE().getHadoopShim(); + + assertSame(shim.getClass(), HadoopShim1.class); + } + +} http://git-wip-us.apache.org/repos/asf/gora/blob/b51cdeea/gora-shims-hadoop2/pom.xml ---------------------------------------------------------------------- diff --git a/gora-shims-hadoop2/pom.xml b/gora-shims-hadoop2/pom.xml new file mode 100644 index 0000000..1ff9776 --- /dev/null +++ b/gora-shims-hadoop2/pom.xml @@ -0,0 +1,43 @@ + <!-- Licensed to the Apache Software Foundation (ASF) under one or more contributor + license agreements. See the NOTICE file distributed with this work for additional + information regarding copyright ownership. The ASF licenses this file to + You under the Apache License, Version 2.0 (the "License"); you may not use + this file except in compliance with the License. You may obtain a copy of + the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required + by applicable law or agreed to in writing, software distributed under the + License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS + OF ANY KIND, either express or implied. See the License for the specific + language governing permissions and limitations under the License. --> +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <modelVersion>4.0.0</modelVersion> + + <parent> + <groupId>org.apache.gora</groupId> + <artifactId>gora</artifactId> + <version>0.5-SNAPSHOT</version> + </parent> + <artifactId>gora-shims-hadoop2</artifactId> + <packaging>bundle</packaging> + + <name>Apache Gora :: Shims Hadoop 2.x</name> + + <dependencies> + <dependency> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-client</artifactId> + <version>${hadoop-2.version}</version> + <optional>true</optional> + </dependency> + <dependency> + <groupId>org.apache.gora</groupId> + <artifactId>gora-shims-hadoop</artifactId> + <version>${project.version}</version> + </dependency> + </dependencies> +</project> http://git-wip-us.apache.org/repos/asf/gora/blob/b51cdeea/gora-shims-hadoop2/src/main/java/org/apache/gora/shims/hadoop2/HadoopShim2.java ---------------------------------------------------------------------- diff --git a/gora-shims-hadoop2/src/main/java/org/apache/gora/shims/hadoop2/HadoopShim2.java b/gora-shims-hadoop2/src/main/java/org/apache/gora/shims/hadoop2/HadoopShim2.java new file mode 100644 index 0000000..527ff36 --- /dev/null +++ b/gora-shims-hadoop2/src/main/java/org/apache/gora/shims/hadoop2/HadoopShim2.java @@ -0,0 +1,56 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.gora.shims.hadoop2; + +import java.io.IOException; + +import org.apache.gora.shims.hadoop.HadoopShim; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.mapreduce.Job; +import org.apache.hadoop.mapreduce.JobContext; +import org.apache.hadoop.mapreduce.task.JobContextImpl; + +/** + * Shim for Hadoop 2.x. + */ +public class HadoopShim2 extends HadoopShim { + + public HadoopShim2() { + super(); + } + + /** + * {@inheritDoc} + * + * Use the Hadoop 2.x way of creating a {@link Job} object. + */ + public Job createJob(Configuration configuration) throws IOException { + Job instance = Job.getInstance(configuration); + return instance; + } + + /** + * {@inheritDoc} + * + * Use the Hadoop 2.x way of creating a {@link JobContext} object. + */ + public JobContext createJobContext(Configuration configuration) { + return new JobContextImpl(configuration, null); + } + +} http://git-wip-us.apache.org/repos/asf/gora/blob/b51cdeea/gora-shims-hadoop2/src/test/java/org/apache/gora/shims/hadoop2/TestHadoopShim2.java ---------------------------------------------------------------------- diff --git a/gora-shims-hadoop2/src/test/java/org/apache/gora/shims/hadoop2/TestHadoopShim2.java b/gora-shims-hadoop2/src/test/java/org/apache/gora/shims/hadoop2/TestHadoopShim2.java new file mode 100644 index 0000000..e97004d --- /dev/null +++ b/gora-shims-hadoop2/src/test/java/org/apache/gora/shims/hadoop2/TestHadoopShim2.java @@ -0,0 +1,18 @@ +package org.apache.gora.shims.hadoop2; + +import org.apache.gora.shims.hadoop.HadoopShim; +import org.apache.gora.shims.hadoop.HadoopShimFactory; +import org.junit.Test; + +import static org.junit.Assert.*; + +public class TestHadoopShim2 { + + @Test + public void testCorrectVersion() { + HadoopShim shim = HadoopShimFactory.INSTANCE().getHadoopShim(); + + assertSame(shim.getClass(), HadoopShim2.class); + } + +} http://git-wip-us.apache.org/repos/asf/gora/blob/b51cdeea/gora-tutorial/pom.xml ---------------------------------------------------------------------- diff --git a/gora-tutorial/pom.xml b/gora-tutorial/pom.xml index 3461fba..cf97905 100644 --- a/gora-tutorial/pom.xml +++ b/gora-tutorial/pom.xml @@ -117,9 +117,10 @@ </dependency--> <!-- Hadoop Dependencies --> - <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-core</artifactId> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-common</artifactId> + <version>${hadoop-2.version}</version> </dependency> <dependency> http://git-wip-us.apache.org/repos/asf/gora/blob/b51cdeea/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index 3362329..c5ed7d8 100644 --- a/pom.xml +++ b/pom.xml @@ -573,12 +573,17 @@ <module>gora-core</module> <module>gora-accumulo</module> <module>gora-cassandra</module> - <module>gora-goraci</module> + <module>gora-goraci</module> + <module>gora-hbase</module> <!-- module>gora-lucene</module --> <!--module>gora-dynamodb</module --> <!--module>gora-sql</module --> <module>gora-mongodb</module> + <module>gora-shims-hadoop</module> + <module>gora-shims-hadoop1</module> + <module>gora-shims-hadoop2</module> + <module>gora-shims-distribution</module> <module>gora-solr</module> <module>gora-tutorial</module> <module>sources-dist</module> @@ -591,7 +596,8 @@ <avro.version>1.7.6</avro.version> <jackson.version>1.6.9</jackson.version> <!-- Hadoop Dependencies --> - <hadoop.version>1.0.1</hadoop.version> + <hadoop-1.version>1.0.1</hadoop-1.version> + <hadoop-2.version>2.4.0</hadoop-2.version> <hadoop.test.version>1.0.1</hadoop.test.version> <hbase.version>0.94.14</hbase.version> <cxf-rt-frontend-jaxrs.version>2.5.2</cxf-rt-frontend-jaxrs.version> @@ -766,7 +772,7 @@ <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-core</artifactId> - <version>${hadoop.version}</version> + <version>${hadoop-1.version}</version> <exclusions> <!-- jackson is conflicting with the Avro dep --> <exclusion> @@ -795,6 +801,23 @@ </exclusion> </exclusions> </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-common</artifactId> + <version>${hadoop-2.version}</version> + <classifier>tests</classifier> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-mapreduce-client-jobclient</artifactId> + <version>${hadoop-2.version}</version> + <classifier>tests</classifier> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-mapreduce-client-core</artifactId> + <version>${hadoop-2.version}</version> + </dependency> <dependency> <groupId>org.apache.cxf</groupId>
