Updated Branches: refs/heads/master cbc7c2fb3 -> 890e0086a
http://git-wip-us.apache.org/repos/asf/crunch/blob/890e0086/crunch/src/test/java/org/apache/crunch/types/writable/WritablesTest.java ---------------------------------------------------------------------- diff --git a/crunch/src/test/java/org/apache/crunch/types/writable/WritablesTest.java b/crunch/src/test/java/org/apache/crunch/types/writable/WritablesTest.java deleted file mode 100644 index 5396fba..0000000 --- a/crunch/src/test/java/org/apache/crunch/types/writable/WritablesTest.java +++ /dev/null @@ -1,256 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.crunch.types.writable; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNotSame; -import static org.junit.Assert.assertSame; - -import java.io.DataInput; -import java.io.DataOutput; -import java.io.IOException; -import java.nio.ByteBuffer; -import java.util.Collection; - -import org.apache.crunch.Pair; -import org.apache.crunch.Tuple3; -import org.apache.crunch.Tuple4; -import org.apache.crunch.TupleN; -import org.apache.crunch.types.PTableType; -import org.apache.crunch.types.PType; -import org.apache.hadoop.io.BooleanWritable; -import org.apache.hadoop.io.BytesWritable; -import org.apache.hadoop.io.DoubleWritable; -import org.apache.hadoop.io.FloatWritable; -import org.apache.hadoop.io.IntWritable; -import org.apache.hadoop.io.LongWritable; -import org.apache.hadoop.io.NullWritable; -import org.apache.hadoop.io.Text; -import org.apache.hadoop.io.Writable; -import org.junit.Test; - -import com.google.common.collect.Lists; - -public class WritablesTest { - - @Test - public void testNulls() throws Exception { - Void n = null; - NullWritable nw = NullWritable.get(); - testInputOutputFn(Writables.nulls(), n, nw); - } - - @Test - public void testStrings() throws Exception { - String s = "abc"; - Text text = new Text(s); - testInputOutputFn(Writables.strings(), s, text); - } - - @Test - public void testInts() throws Exception { - int j = 55; - IntWritable w = new IntWritable(j); - testInputOutputFn(Writables.ints(), j, w); - } - - @Test - public void testLongs() throws Exception { - long j = 55; - LongWritable w = new LongWritable(j); - testInputOutputFn(Writables.longs(), j, w); - } - - @Test - public void testFloats() throws Exception { - float j = 55.5f; - FloatWritable w = new FloatWritable(j); - testInputOutputFn(Writables.floats(), j, w); - } - - @Test - public void testDoubles() throws Exception { - double j = 55.5d; - DoubleWritable w = new DoubleWritable(j); - testInputOutputFn(Writables.doubles(), j, w); - } - - @Test - public void testBoolean() throws Exception { - boolean j = false; - BooleanWritable w = new BooleanWritable(j); - testInputOutputFn(Writables.booleans(), j, w); - } - - @Test - public void testBytes() throws Exception { - byte[] bytes = new byte[] { 17, 26, -98 }; - BytesWritable bw = new BytesWritable(bytes); - ByteBuffer bb = ByteBuffer.wrap(bytes); - testInputOutputFn(Writables.bytes(), bb, bw); - } - - @Test - public void testCollections() throws Exception { - String s = "abc"; - Collection<String> j = Lists.newArrayList(); - j.add(s); - GenericArrayWritable<Text> w = new GenericArrayWritable<Text>(Text.class); - w.set(new Text[] { new Text(s) }); - testInputOutputFn(Writables.collections(Writables.strings()), j, w); - } - - @Test - public void testPairs() throws Exception { - Pair<String, String> j = Pair.of("a", "b"); - TupleWritable w = new TupleWritable(new Text[] { new Text("a"), new Text("b"), }); - w.setWritten(0); - w.setWritten(1); - testInputOutputFn(Writables.pairs(Writables.strings(), Writables.strings()), j, w); - } - - @Test - public void testNestedTables() throws Exception { - PTableType<Long, Long> pll = Writables.tableOf(Writables.longs(), Writables.longs()); - PTableType<Pair<Long, Long>, String> nest = Writables.tableOf(pll, Writables.strings()); - assertNotNull(nest); - } - - @Test - public void testPairEquals() throws Exception { - PType<Pair<Long, ByteBuffer>> t1 = Writables.pairs(Writables.longs(), Writables.bytes()); - PType<Pair<Long, ByteBuffer>> t2 = Writables.pairs(Writables.longs(), Writables.bytes()); - assertEquals(t1, t2); - assertEquals(t1.hashCode(), t2.hashCode()); - } - - @Test - @SuppressWarnings("rawtypes") - public void testTriples() throws Exception { - Tuple3 j = Tuple3.of("a", "b", "c"); - TupleWritable w = new TupleWritable(new Text[] { new Text("a"), new Text("b"), new Text("c"), }); - w.setWritten(0); - w.setWritten(1); - w.setWritten(2); - WritableType<?, ?> wt = Writables.triples(Writables.strings(), Writables.strings(), Writables.strings()); - testInputOutputFn(wt, j, w); - } - - @Test - @SuppressWarnings("rawtypes") - public void testQuads() throws Exception { - Tuple4 j = Tuple4.of("a", "b", "c", "d"); - TupleWritable w = new TupleWritable(new Text[] { new Text("a"), new Text("b"), new Text("c"), new Text("d"), }); - w.setWritten(0); - w.setWritten(1); - w.setWritten(2); - w.setWritten(3); - WritableType<?, ?> wt = Writables.quads(Writables.strings(), Writables.strings(), Writables.strings(), - Writables.strings()); - testInputOutputFn(wt, j, w); - } - - @Test - public void testTupleN() throws Exception { - TupleN j = new TupleN("a", "b", "c", "d", "e"); - TupleWritable w = new TupleWritable(new Text[] { new Text("a"), new Text("b"), new Text("c"), new Text("d"), - new Text("e"), }); - w.setWritten(0); - w.setWritten(1); - w.setWritten(2); - w.setWritten(3); - w.setWritten(4); - WritableType<?, ?> wt = Writables.tuples(Writables.strings(), Writables.strings(), Writables.strings(), - Writables.strings(), Writables.strings()); - testInputOutputFn(wt, j, w); - } - - protected static class TestWritable implements Writable { - String left; - int right; - - @Override - public void write(DataOutput out) throws IOException { - out.writeUTF(left); - out.writeInt(right); - } - - @Override - public void readFields(DataInput in) throws IOException { - left = in.readUTF(); - right = in.readInt(); - } - - @Override - public boolean equals(Object obj) { - if (this == obj) - return true; - if (obj == null) - return false; - if (getClass() != obj.getClass()) - return false; - TestWritable other = (TestWritable) obj; - if (left == null) { - if (other.left != null) - return false; - } else if (!left.equals(other.left)) - return false; - if (right != other.right) - return false; - return true; - } - - } - - @Test - public void testRecords() throws Exception { - TestWritable j = new TestWritable(); - j.left = "a"; - j.right = 1; - TestWritable w = new TestWritable(); - w.left = "a"; - w.right = 1; - WritableType<?, ?> wt = Writables.records(TestWritable.class); - testInputOutputFn(wt, j, w); - } - - @Test - public void testTableOf() throws Exception { - Pair<String, String> j = Pair.of("a", "b"); - Pair<Text, Text> w = Pair.of(new Text("a"), new Text("b")); - WritableTableType<String, String> wtt = Writables.tableOf(Writables.strings(), Writables.strings()); - testInputOutputFn(wtt, j, w); - } - - @Test - public void testRegister() throws Exception { - WritableType<TestWritable, TestWritable> wt = Writables.writables(TestWritable.class); - Writables.register(TestWritable.class, wt); - assertSame(Writables.records(TestWritable.class), wt); - } - - @SuppressWarnings({ "unchecked", "rawtypes" }) - protected static void testInputOutputFn(PType ptype, Object java, Object writable) { - ptype.getInputMapFn().initialize(); - ptype.getOutputMapFn().initialize(); - assertEquals(java, ptype.getInputMapFn().map(writable)); - assertEquals(writable, ptype.getOutputMapFn().map(java)); - } - -} http://git-wip-us.apache.org/repos/asf/crunch/blob/890e0086/crunch/src/test/java/org/apache/crunch/util/DistCacheTest.java ---------------------------------------------------------------------- diff --git a/crunch/src/test/java/org/apache/crunch/util/DistCacheTest.java b/crunch/src/test/java/org/apache/crunch/util/DistCacheTest.java deleted file mode 100644 index 6784f14..0000000 --- a/crunch/src/test/java/org/apache/crunch/util/DistCacheTest.java +++ /dev/null @@ -1,156 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.crunch.util; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -import java.io.IOException; -import java.util.HashSet; -import java.util.Set; - -import org.apache.commons.lang.StringUtils; -import org.apache.hadoop.conf.Configuration; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; - -public class DistCacheTest { - - // A temporary folder used to hold files created for the test. - @Rule - public TemporaryFolder testFolder = new TemporaryFolder(); - - // A configuration and lists of paths to use in tests. - private Configuration testConf; - private String[] testFilePaths; - private String[] testFileQualifiedPaths; - - /** - * Setup resources for tests. These include: - * <ol> - * <li>A Hadoop configuration. - * <li>A directory of temporary files that includes 3 .jar files and 1 other - * file. - * <li>Arrays containing the canonical paths and qualified paths to the test - * files. - * </ol> - */ - @Before - public void setup() throws IOException { - // Create a configuration for tests. - testConf = new Configuration(); - - // Create the test files and add their paths to the list of test file paths. - testFilePaths = new String[3]; - testFilePaths[0] = testFolder.newFile("jar1.jar").getCanonicalPath(); - testFilePaths[1] = testFolder.newFile("jar2.jar").getCanonicalPath(); - testFilePaths[2] = testFolder.newFile("jar3.jar").getCanonicalPath(); - testFolder.newFile("notJar.other"); - - // Populate a list of qualified paths from the test file paths. - testFileQualifiedPaths = new String[3]; - for (int i = 0; i < testFilePaths.length; i++) { - testFileQualifiedPaths[i] = "file:" + testFilePaths[i]; - } - } - - /** - * Tests adding jars one-by-one to a job's configuration. - * - * @throws IOException - * If there is a problem adding the jars. - */ - @Test - public void testAddJar() throws IOException { - // Add each valid jar path to the distributed cache configuration, and - // verify each was - // added correctly in turn. - for (int i = 0; i < testFilePaths.length; i++) { - DistCache.addJarToDistributedCache(testConf, testFilePaths[i]); - assertEquals("tmpjars configuration var does not contain expected value.", - StringUtils.join(testFileQualifiedPaths, ",", 0, i + 1), testConf.get("tmpjars")); - } - } - - /** - * Tests that attempting to add the path to a jar that does not exist to the - * configuration throws an exception. - * - * @throws IOException - * If the added jar path does not exist. This exception is expected. - */ - @Test(expected = IOException.class) - public void testAddJarThatDoesntExist() throws IOException { - DistCache.addJarToDistributedCache(testConf, "/garbage/doesntexist.jar"); - } - - /** - * Tests that adding a directory of jars to the configuration works as - * expected. .jar files under the added directory should be added to the - * configuration, and all other files should be skipped. - * - * @throws IOException - * If there is a problem adding the jar directory to the - * configuration. - */ - @Test - public void testAddJarDirectory() throws IOException { - DistCache.addJarDirToDistributedCache(testConf, testFolder.getRoot().getCanonicalPath()); - // Throw the added jar paths in a set to detect duplicates. - String[] splitJarPaths = StringUtils.split(testConf.get("tmpjars"), ","); - Set<String> addedJarPaths = new HashSet<String>(); - for (String path : splitJarPaths) { - addedJarPaths.add(path); - } - assertEquals("Incorrect number of jar paths added.", testFilePaths.length, addedJarPaths.size()); - - // Ensure all expected paths were added. - for (int i = 0; i < testFileQualifiedPaths.length; i++) { - assertTrue("Expected jar path missing from jar paths added to tmpjars: " + testFileQualifiedPaths[i], - addedJarPaths.contains(testFileQualifiedPaths[i])); - } - } - - /** - * Tests that adding a jar directory that does not exist to the configuration - * throws an exception. - * - * @throws IOException - * If the added jar directory does not exist. This exception is - * expected. - */ - @Test(expected = IOException.class) - public void testAddJarDirectoryThatDoesntExist() throws IOException { - DistCache.addJarDirToDistributedCache(testConf, "/garbage/doesntexist"); - } - - /** - * Tests that adding a jar directory that is not a directory to the - * configuration throws an exception. - * - * @throws IOException - * If the added jar directory is not a directory. This exception is - * expected. - */ - @Test(expected = IOException.class) - public void testAddJarDirectoryNotDirectory() throws IOException { - DistCache.addJarDirToDistributedCache(testConf, testFilePaths[0]); - } -} http://git-wip-us.apache.org/repos/asf/crunch/blob/890e0086/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index d5f90f2..71f5e0f 100644 --- a/pom.xml +++ b/pom.xml @@ -44,7 +44,7 @@ under the License. </prerequisites> <modules> - <module>crunch</module> + <module>crunch-core</module> <module>crunch-hbase</module> <module>crunch-test</module> <module>crunch-contrib</module> @@ -103,7 +103,7 @@ under the License. <dependencies> <dependency> <groupId>org.apache.crunch</groupId> - <artifactId>crunch</artifactId> + <artifactId>crunch-core</artifactId> <version>${project.version}</version> </dependency>
