http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/5accc9ac/src/test/java/org/apache/crunch/types/writable/WritablesTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/crunch/types/writable/WritablesTest.java b/src/test/java/org/apache/crunch/types/writable/WritablesTest.java new file mode 100644 index 0000000..593b773 --- /dev/null +++ b/src/test/java/org/apache/crunch/types/writable/WritablesTest.java @@ -0,0 +1,279 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.crunch.types.writable; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNotSame; +import static org.junit.Assert.assertSame; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Collection; + +import org.apache.hadoop.io.BooleanWritable; +import org.apache.hadoop.io.BytesWritable; +import org.apache.hadoop.io.DoubleWritable; +import org.apache.hadoop.io.FloatWritable; +import org.apache.hadoop.io.IntWritable; +import org.apache.hadoop.io.LongWritable; +import org.apache.hadoop.io.NullWritable; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.io.Writable; +import org.junit.Test; + +import org.apache.crunch.Pair; +import org.apache.crunch.Tuple3; +import org.apache.crunch.Tuple4; +import org.apache.crunch.TupleN; +import org.apache.crunch.types.PTableType; +import org.apache.crunch.types.PType; +import com.google.common.collect.Lists; + +public class WritablesTest { + + @Test + public void testNulls() throws Exception { + Void n = null; + NullWritable nw = NullWritable.get(); + testInputOutputFn(Writables.nulls(), n, nw); + } + + @Test + public void testStrings() throws Exception { + String s = "abc"; + Text text = new Text(s); + testInputOutputFn(Writables.strings(), s, text); + } + + @Test + public void testInts() throws Exception { + int j = 55; + IntWritable w = new IntWritable(j); + testInputOutputFn(Writables.ints(), j, w); + } + + @Test + public void testLongs() throws Exception { + long j = 55; + LongWritable w = new LongWritable(j); + testInputOutputFn(Writables.longs(), j, w); + } + + @Test + public void testFloats() throws Exception { + float j = 55.5f; + FloatWritable w = new FloatWritable(j); + testInputOutputFn(Writables.floats(), j, w); + } + + @Test + public void testDoubles() throws Exception { + double j = 55.5d; + DoubleWritable w = new DoubleWritable(j); + testInputOutputFn(Writables.doubles(), j, w); + } + + @Test + public void testBoolean() throws Exception { + boolean j = false; + BooleanWritable w = new BooleanWritable(j); + testInputOutputFn(Writables.booleans(), j, w); + } + + @Test + public void testBytes() throws Exception { + byte[] bytes = new byte[] { 17, 26, -98 }; + BytesWritable bw = new BytesWritable(bytes); + ByteBuffer bb = ByteBuffer.wrap(bytes); + testInputOutputFn(Writables.bytes(), bb, bw); + } + + @Test + public void testCollections() throws Exception { + String s = "abc"; + Collection<String> j = Lists.newArrayList(); + j.add(s); + GenericArrayWritable<Text> w = new GenericArrayWritable<Text>(Text.class); + w.set(new Text[]{ + new Text(s) + }); + testInputOutputFn(Writables.collections(Writables.strings()), j, w); + } + + @Test + public void testPairs() throws Exception { + Pair<String, String> j = Pair.of("a", "b"); + TupleWritable w = new TupleWritable(new Text[] { + new Text("a"), + new Text("b"), + }); + w.setWritten(0); + w.setWritten(1); + testInputOutputFn(Writables.pairs(Writables.strings(), Writables.strings()), j, w); + } + + @Test + public void testNestedTables() throws Exception { + PTableType<Long, Long> pll = Writables.tableOf(Writables.longs(), Writables.longs()); + PTableType<Pair<Long, Long>, String> nest = Writables.tableOf(pll, Writables.strings()); + assertNotNull(nest); + } + + @Test + public void testPairEquals() throws Exception { + PType<Pair<Long, ByteBuffer>> t1 = Writables.pairs(Writables.longs(), Writables.bytes()); + PType<Pair<Long, ByteBuffer>> t2 = Writables.pairs(Writables.longs(), Writables.bytes()); + assertEquals(t1, t2); + assertEquals(t1.hashCode(), t2.hashCode()); + } + + @Test + @SuppressWarnings("rawtypes") + public void testTriples() throws Exception { + Tuple3 j = Tuple3.of("a", "b", "c"); + TupleWritable w = new TupleWritable(new Text[] { + new Text("a"), + new Text("b"), + new Text("c"), + }); + w.setWritten(0); + w.setWritten(1); + w.setWritten(2); + WritableType<?, ?> wt = Writables.triples(Writables.strings(), + Writables.strings(), Writables.strings()); + testInputOutputFn(wt, j, w); + } + + @Test + @SuppressWarnings("rawtypes") + public void testQuads() throws Exception { + Tuple4 j = Tuple4.of("a", "b", "c", "d"); + TupleWritable w = new TupleWritable(new Text[] { + new Text("a"), + new Text("b"), + new Text("c"), + new Text("d"), + }); + w.setWritten(0); + w.setWritten(1); + w.setWritten(2); + w.setWritten(3); + WritableType<?, ?> wt = Writables.quads(Writables.strings(), Writables.strings(), + Writables.strings(), Writables.strings()); + testInputOutputFn(wt, j, w); + } + + @Test + public void testTupleN() throws Exception { + TupleN j = new TupleN("a", "b", "c", "d", "e"); + TupleWritable w = new TupleWritable(new Text[] { + new Text("a"), + new Text("b"), + new Text("c"), + new Text("d"), + new Text("e"), + }); + w.setWritten(0); + w.setWritten(1); + w.setWritten(2); + w.setWritten(3); + w.setWritten(4); + WritableType<?, ?> wt = Writables.tuples(Writables.strings(), Writables.strings(), + Writables.strings(), Writables.strings(), Writables.strings()); + testInputOutputFn(wt, j, w); + } + + protected static class TestWritable implements Writable { + String left; + int right; + @Override + public void write(DataOutput out) throws IOException { + out.writeUTF(left); + out.writeInt(right); + } + @Override + public void readFields(DataInput in) throws IOException { + left = in.readUTF(); + right = in.readInt(); + } + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + TestWritable other = (TestWritable) obj; + if (left == null) { + if (other.left != null) + return false; + } else if (!left.equals(other.left)) + return false; + if (right != other.right) + return false; + return true; + } + + } + @Test + public void testRecords() throws Exception { + TestWritable j = new TestWritable(); + j.left = "a"; + j.right = 1; + TestWritable w = new TestWritable(); + w.left = "a"; + w.right = 1; + WritableType<?, ?> wt = Writables.records(TestWritable.class); + testInputOutputFn(wt, j, w); + } + + @Test + public void testTableOf() throws Exception { + Pair<String, String> j = Pair.of("a", "b"); + Pair<Text, Text> w = Pair.of(new Text("a"), new Text("b")); + WritableTableType<String, String> wtt = Writables.tableOf(Writables.strings(), Writables.strings()); + testInputOutputFn(wtt, j, w); + } + + @Test + public void testRegister() throws Exception { + WritableType<TestWritable, TestWritable> wt = Writables.writables(TestWritable.class); + Writables.register(TestWritable.class, wt); + assertSame(Writables.records(TestWritable.class), wt); + } + + @SuppressWarnings({"unchecked", "rawtypes"}) + protected static void testInputOutputFn(PType ptype, Object java, Object writable) { + ptype.getInputMapFn().initialize(); + ptype.getOutputMapFn().initialize(); + assertEquals(java, ptype.getInputMapFn().map(writable)); + assertEquals(writable, ptype.getOutputMapFn().map(java)); + } + + @Test + public void testDeepCopy() { + Text text = new Text("Test"); + Text copiedText = Writables.deepCopy(text, Text.class); + assertEquals(text, copiedText); + assertNotSame(text, copiedText); + } +}
http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/5accc9ac/src/test/java/org/apache/crunch/util/DistCacheTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/crunch/util/DistCacheTest.java b/src/test/java/org/apache/crunch/util/DistCacheTest.java new file mode 100644 index 0000000..eda45c4 --- /dev/null +++ b/src/test/java/org/apache/crunch/util/DistCacheTest.java @@ -0,0 +1,144 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.crunch.util; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.io.IOException; +import java.util.HashSet; +import java.util.Set; + +import org.apache.commons.lang.StringUtils; +import org.apache.hadoop.conf.Configuration; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +public class DistCacheTest { + + // A temporary folder used to hold files created for the test. + @Rule + public TemporaryFolder testFolder = new TemporaryFolder(); + + // A configuration and lists of paths to use in tests. + private Configuration testConf; + private String[] testFilePaths; + private String[] testFileQualifiedPaths; + + /** + * Setup resources for tests. These include: + * 1. A Hadoop configuration. + * 2. A directory of temporary files that includes 3 .jar files and 1 other file. + * 3. Arrays containing the canonical paths and qualified paths to the test files. + */ + @Before + public void setup() throws IOException { + // Create a configuration for tests. + testConf = new Configuration(); + + // Create the test files and add their paths to the list of test file paths. + testFilePaths = new String[3]; + testFilePaths[0] = testFolder.newFile("jar1.jar").getCanonicalPath(); + testFilePaths[1] = testFolder.newFile("jar2.jar").getCanonicalPath(); + testFilePaths[2] = testFolder.newFile("jar3.jar").getCanonicalPath(); + testFolder.newFile("notJar.other"); + + // Populate a list of qualified paths from the test file paths. + testFileQualifiedPaths = new String[3]; + for (int i = 0; i < testFilePaths.length; i++) { + testFileQualifiedPaths[i] = "file:" + testFilePaths[i]; + } + } + + /** + * Tests adding jars one-by-one to a job's configuration. + * + * @throws IOException If there is a problem adding the jars. + */ + @Test + public void testAddJar() throws IOException { + // Add each valid jar path to the distributed cache configuration, and verify each was + // added correctly in turn. + for (int i = 0; i < testFilePaths.length; i++) { + DistCache.addJarToDistributedCache(testConf, testFilePaths[i]); + assertEquals("tmpjars configuration var does not contain expected value.", + StringUtils.join(testFileQualifiedPaths, ",", 0, i + 1), testConf.get("tmpjars")); + } + } + + /** + * Tests that attempting to add the path to a jar that does not exist to the configuration + * throws an exception. + * + * @throws IOException If the added jar path does not exist. This exception is expected. + */ + @Test(expected = IOException.class) + public void testAddJarThatDoesntExist() throws IOException { + DistCache.addJarToDistributedCache(testConf, "/garbage/doesntexist.jar"); + } + + /** + * Tests that adding a directory of jars to the configuration works as expected. .jar files + * under the added directory should be added to the configuration, + * and all other files should be skipped. + * + * @throws IOException If there is a problem adding the jar directory to the configuration. + */ + @Test + public void testAddJarDirectory() throws IOException { + DistCache.addJarDirToDistributedCache(testConf, testFolder.getRoot().getCanonicalPath()); + // Throw the added jar paths in a set to detect duplicates. + String[] splitJarPaths = StringUtils.split(testConf.get("tmpjars"), ","); + Set<String> addedJarPaths = new HashSet<String>(); + for (String path: splitJarPaths) { + addedJarPaths.add(path); + } + assertEquals("Incorrect number of jar paths added.", testFilePaths.length, + addedJarPaths.size()); + + // Ensure all expected paths were added. + for (int i = 0; i < testFileQualifiedPaths.length; i++) { + assertTrue("Expected jar path missing from jar paths added to tmpjars: " + + testFileQualifiedPaths[i], addedJarPaths.contains(testFileQualifiedPaths[i])); + } + } + + /** + * Tests that adding a jar directory that does not exist to the configuration throws an + * exception. + * + * @throws IOException If the added jar directory does not exist. This exception is expected. + */ + @Test(expected = IOException.class) + public void testAddJarDirectoryThatDoesntExist() throws IOException { + DistCache.addJarDirToDistributedCache(testConf, "/garbage/doesntexist"); + } + + /** + * Tests that adding a jar directory that is not a directory to the configuration throws an + * exception. + * + * @throws IOException If the added jar directory is not a directory. This exception is expected. + */ + @Test(expected = IOException.class) + public void testAddJarDirectoryNotDirectory() throws IOException { + DistCache.addJarDirToDistributedCache(testConf, testFilePaths[0]); + } +}
