Repository: incubator-datafu Updated Branches: refs/heads/master 4aa2ef2a4 -> 778bef1eb
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/test/pig/datafu/test/pig/urls/UserAgentTest.java ---------------------------------------------------------------------- diff --git a/test/pig/datafu/test/pig/urls/UserAgentTest.java b/test/pig/datafu/test/pig/urls/UserAgentTest.java deleted file mode 100644 index 82515ad..0000000 --- a/test/pig/datafu/test/pig/urls/UserAgentTest.java +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package datafu.test.pig.urls; - -import org.adrianwalker.multilinestring.Multiline; -import org.apache.pig.pigunit.PigTest; -import org.testng.annotations.Test; - -import datafu.test.pig.PigTests; - -public class UserAgentTest extends PigTests -{ - - /** - register $JAR_PATH - - define UserAgentClassify datafu.pig.urls.UserAgentClassify(); - - data = load 'input' as (usr_agent:chararray); - data_out = foreach data generate UserAgentClassify(usr_agent) as class; - --describe data_out; - store data_out into 'output'; - */ - @Multiline private String userAgentTest; - - @Test - public void userAgentTest() throws Exception - { - PigTest test = createPigTestFromString(userAgentTest); - - String[] input = { - "Mozilla/5.0 (iPhone; U; CPU iPhone OS 4_3_3 like Mac OS X; en-us) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8J2 Safari/6533.18.5", - "Mozilla/5.0 (compatible; Konqueror/3.5; Linux; X11; de) KHTML/3.5.2 (like Gecko) Kubuntu 6.06 Dapper", - "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:2.2a1pre) Gecko/20110331 Firefox/4.2a1pre Fennec/4.1a1pre", - "Opera/9.00 (X11; Linux i686; U; en)", - "Wget/1.10.2", - "Opera/9.80 (Android; Linux; Opera Mobi/ADR-1012221546; U; pl) Presto/2.7.60 Version/10.5", - "Mozilla/5.0 (Linux; U; Android 2.2; en-us; DROID2 Build/VZW) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1" - }; - - String[] output = { - "(mobile)", - "(desktop)", - "(mobile)", - "(desktop)", - "(desktop)", - "(mobile)", - "(mobile)", - }; - - test.assertOutput("data",input,"data_out",output); - } - -} http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/test/pig/datafu/test/pig/util/AliasEvalFuncTest.java ---------------------------------------------------------------------- diff --git a/test/pig/datafu/test/pig/util/AliasEvalFuncTest.java b/test/pig/datafu/test/pig/util/AliasEvalFuncTest.java deleted file mode 100644 index 3b23d35..0000000 --- a/test/pig/datafu/test/pig/util/AliasEvalFuncTest.java +++ /dev/null @@ -1,95 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package datafu.test.pig.util; - -import static org.testng.Assert.*; - -import java.util.List; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Iterator; -import java.io.IOException; - -import org.apache.pig.data.Tuple; -import org.apache.pig.data.DataBag; -import org.apache.pig.data.TupleFactory; -import org.apache.pig.data.BagFactory; -import org.apache.pig.pigunit.PigTest; -import org.apache.pig.data.DataType; -import org.apache.pig.impl.logicalLayer.schema.Schema; -import org.apache.pig.impl.logicalLayer.schema.Schema.FieldSchema; - -import org.testng.Assert; -import org.testng.annotations.Test; - -import datafu.test.pig.PigTests; -import datafu.pig.util.AliasableEvalFunc; - -public class AliasEvalFuncTest extends PigTests -{ - static class ReportBuilder extends AliasableEvalFunc<DataBag> { - static final String ORDERED_ROUTES = "orderedRoutes"; - - public DataBag exec(Tuple input) throws IOException { - DataBag inputBag = getBag(input, ORDERED_ROUTES); - DataBag outputBag = BagFactory.getInstance().newDefaultBag(); - for(Iterator<Tuple> tupleIter = inputBag.iterator(); tupleIter.hasNext(); ) { - outputBag.add(tupleIter.next()); - } - return outputBag; - } - - public Schema getOutputSchema(Schema input) { - try { - Schema bagSchema = input.getField(0).schema; - Schema outputSchema = new Schema(new Schema.FieldSchema(getSchemaName(this.getClass() - .getName() - .toLowerCase(), input), - bagSchema, - DataType.BAG)); - return outputSchema; - } catch (Exception ex) { - return null; - } - } - } - - @Test - public void getBagTest() throws Exception - { - ReportBuilder udf = new ReportBuilder(); - udf.setUDFContextSignature("test"); - List<Schema.FieldSchema> fieldSchemaList = new ArrayList<Schema.FieldSchema>(); - fieldSchemaList.add(new Schema.FieldSchema("msisdn", DataType.LONG)); - fieldSchemaList.add(new Schema.FieldSchema("ts", DataType.INTEGER)); - fieldSchemaList.add(new Schema.FieldSchema("center_lon", DataType.DOUBLE)); - fieldSchemaList.add(new Schema.FieldSchema("center_lat", DataType.DOUBLE)); - Schema schemaTuple = new Schema(fieldSchemaList); - Schema schemaBag = new Schema(new Schema.FieldSchema(ReportBuilder.ORDERED_ROUTES, schemaTuple, DataType.BAG)); - udf.outputSchema(schemaBag); - - Tuple inputTuple = TupleFactory.getInstance().newTuple(); - DataBag inputBag = BagFactory.getInstance().newDefaultBag(); - inputBag.add(TupleFactory.getInstance().newTuple(Arrays.asList(71230000000L, 1382351612, 10.697, 20.713))); - inputTuple.append(inputBag); - DataBag outputBag = udf.exec(inputTuple); - Assert.assertEquals(inputBag, outputBag); - } -} http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/test/pig/datafu/test/pig/util/AssertTests.java ---------------------------------------------------------------------- diff --git a/test/pig/datafu/test/pig/util/AssertTests.java b/test/pig/datafu/test/pig/util/AssertTests.java deleted file mode 100644 index bf558a7..0000000 --- a/test/pig/datafu/test/pig/util/AssertTests.java +++ /dev/null @@ -1,131 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package datafu.test.pig.util; - -import static org.testng.Assert.*; - -import java.util.List; - -import org.adrianwalker.multilinestring.Multiline; -import org.apache.pig.data.Tuple; -import org.apache.pig.pigunit.PigTest; -import org.testng.Assert; -import org.testng.annotations.Test; - -import datafu.test.pig.PigTests; - -public class AssertTests extends PigTests -{ - /** - register $JAR_PATH - - define ASRT datafu.pig.util.AssertUDF(); - - data = LOAD 'input' AS (val:INT); - - data2 = FILTER data BY ASRT(val,'assertion appears to have failed, doh!'); - - STORE data2 INTO 'output'; - */ - @Multiline private static String assertWithMessage; - - @Test - public void shouldAssertWithMessageOnZero() throws Exception - { - try - { - PigTest test = createPigTestFromString(assertWithMessage); - - this.writeLinesToFile("input", "0"); - - test.runScript(); - - this.getLinesForAlias(test, "data2"); - - fail("test should have failed, but it didn't"); - } - catch (Exception e) - { - } - } - - @Test - public void shouldNotAssertWithMessageOnOne() throws Exception - { - PigTest test = createPigTestFromString(assertWithMessage); - - this.writeLinesToFile("input", "1"); - - test.runScript(); - - List<Tuple> result = this.getLinesForAlias(test, "data2"); - Assert.assertEquals(result.size(), 1); - Assert.assertEquals(result.get(0).size(), 1); - Assert.assertEquals(result.get(0).get(0), 1); - } - - /** - register $JAR_PATH - - define ASRT datafu.pig.util.AssertUDF(); - - data = LOAD 'input' AS (val:INT); - - data2 = FILTER data BY ASRT(val); - - STORE data2 INTO 'output'; - */ - @Multiline private static String assertWithoutMessage; - - @Test - public void shouldAssertWithoutMessageOnZero() throws Exception - { - try - { - PigTest test = createPigTestFromString(assertWithoutMessage); - - this.writeLinesToFile("input", "0"); - - test.runScript(); - - this.getLinesForAlias(test, "data2"); - - fail("test should have failed, but it didn't"); - } - catch (Exception e) - { - } - } - - @Test - public void shouldNotAssertWithoutMessageOnOne() throws Exception - { - PigTest test = createPigTestFromString(assertWithoutMessage); - - this.writeLinesToFile("input", "1"); - - test.runScript(); - - List<Tuple> result = this.getLinesForAlias(test, "data2"); - Assert.assertEquals(result.size(), 1); - Assert.assertEquals(result.get(0).size(), 1); - Assert.assertEquals(result.get(0).get(0), 1); - } -} http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/test/pig/datafu/test/pig/util/CoalesceTests.java ---------------------------------------------------------------------- diff --git a/test/pig/datafu/test/pig/util/CoalesceTests.java b/test/pig/datafu/test/pig/util/CoalesceTests.java deleted file mode 100644 index 0dc0f4b..0000000 --- a/test/pig/datafu/test/pig/util/CoalesceTests.java +++ /dev/null @@ -1,532 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package datafu.test.pig.util; - -import java.util.List; - -import junit.framework.Assert; - -import org.adrianwalker.multilinestring.Multiline; -import org.apache.pig.data.Tuple; -import org.apache.pig.impl.logicalLayer.FrontendException; -import org.apache.pig.pigunit.PigTest; -import org.joda.time.DateTime; -import org.joda.time.DateTimeZone; -import org.testng.annotations.Test; - -import datafu.test.pig.PigTests; - -public class CoalesceTests extends PigTests -{ - /** - register $JAR_PATH - - define COALESCE datafu.pig.util.Coalesce(); - - data = LOAD 'input' using PigStorage(',') AS (testcase:INT,val1:INT,val2:INT,val3:INT); - - data2 = FOREACH data GENERATE testcase, COALESCE(val1,val2,val3) as result; - - describe data2; - - data3 = FOREACH data2 GENERATE testcase, result; - - STORE data3 INTO 'output'; - */ - @Multiline private static String coalesceIntTest; - - @Test - public void coalesceIntTest() throws Exception - { - PigTest test = createPigTestFromString(coalesceIntTest); - - this.writeLinesToFile("input", "1,1,2,3", - "2,,2,3", - "3,,,3", - "4,,,", - "5,1,,3", - "6,1,,"); - - test.runScript(); - - List<Tuple> lines = this.getLinesForAlias(test, "data3"); - - Assert.assertEquals(6, lines.size()); - for (Tuple t : lines) - { - switch((Integer)t.get(0)) - { - case 1: - Assert.assertEquals(1, t.get(1)); break; - case 2: - Assert.assertEquals(2, t.get(1)); break; - case 3: - Assert.assertEquals(3, t.get(1)); break; - case 4: - Assert.assertEquals(null, t.get(1)); break; - case 5: - Assert.assertEquals(1, t.get(1)); break; - case 6: - Assert.assertEquals(1, t.get(1)); break; - default: - Assert.fail("Did not expect: " + t.get(1)); - } - } - } - - /** - register $JAR_PATH - - define COALESCE datafu.pig.util.Coalesce(); - - data = LOAD 'input' using PigStorage(',') AS (testcase:INT,val1:LONG); - - data2 = FOREACH data GENERATE testcase, COALESCE(val1,100L) as result; - - describe data2; - - data3 = FOREACH data2 GENERATE testcase, result; - - data4 = FOREACH data3 GENERATE testcase, result*100 as result; - - STORE data4 INTO 'output'; - */ - @Multiline private static String coalesceLongTest; - - @Test - public void coalesceLongTest() throws Exception - { - PigTest test = createPigTestFromString(coalesceLongTest); - - this.writeLinesToFile("input", "1,5", - "2,"); - - test.runScript(); - - List<Tuple> lines = this.getLinesForAlias(test, "data4"); - - Assert.assertEquals(2, lines.size()); - for (Tuple t : lines) - { - switch((Integer)t.get(0)) - { - case 1: - Assert.assertEquals(500L, t.get(1)); break; - case 2: - Assert.assertEquals(10000L, t.get(1)); break; - default: - Assert.fail("Did not expect: " + t.get(1)); - } - } - } - - /** - register $JAR_PATH - - define COALESCE datafu.pig.util.Coalesce(); - - data = LOAD 'input' using PigStorage(',') AS (testcase:INT,val1:LONG); - - data2 = FOREACH data GENERATE testcase, COALESCE(val1,100) as result; - - describe data2; - - data3 = FOREACH data2 GENERATE testcase, result; - - data4 = FOREACH data3 GENERATE testcase, result*100 as result; - - STORE data4 INTO 'output'; - */ - @Multiline private static String coalesceCastIntToLongTestFails; - - // The first parameter is a long and the fixed value is an int. - // They cannot be merged without the lazy option. - @Test(expectedExceptions=FrontendException.class) - public void coalesceCastIntToLongTestFails() throws Exception - { - PigTest test = createPigTestFromString(coalesceCastIntToLongTestFails); - - this.writeLinesToFile("input", "1,5", - "2,"); - - test.runScript(); - - List<Tuple> lines = this.getLinesForAlias(test, "data4"); - - Assert.assertEquals(2, lines.size()); - for (Tuple t : lines) - { - switch((Integer)t.get(0)) - { - case 1: - Assert.assertEquals(500L, t.get(1)); break; - case 2: - Assert.assertEquals(10000L, t.get(1)); break; - default: - Assert.fail("Did not expect: " + t.get(1)); - } - } - } - - /** - register $JAR_PATH - - define COALESCE datafu.pig.util.Coalesce('lazy'); - - data = LOAD 'input' using PigStorage(',') AS (testcase:INT,val1:LONG); - - data2 = FOREACH data GENERATE testcase, COALESCE(val1,100) as result; - - describe data2; - - data3 = FOREACH data2 GENERATE testcase, result; - - data4 = FOREACH data3 GENERATE testcase, result*100 as result; - - STORE data4 INTO 'output'; - */ - @Multiline private static String coalesceIntAndLongTest; - - // The first parameter is a long and the fixed value is an int. - // They are merged to a long. - @Test - public void coalesceCastIntToLongTest1() throws Exception - { - PigTest test = createPigTestFromString(coalesceIntAndLongTest); - - this.writeLinesToFile("input", "1,5", - "2,"); - - test.runScript(); - - List<Tuple> lines = this.getLinesForAlias(test, "data4"); - - Assert.assertEquals(2, lines.size()); - for (Tuple t : lines) - { - switch((Integer)t.get(0)) - { - case 1: - Assert.assertEquals(500L, t.get(1)); break; - case 2: - Assert.assertEquals(10000L, t.get(1)); break; - default: - Assert.fail("Did not expect: " + t.get(1)); - } - } - } - - /** - register $JAR_PATH - - define COALESCE datafu.pig.util.Coalesce('lazy'); - - data = LOAD 'input' using PigStorage(',') AS (testcase:INT,val1:INT); - - data2 = FOREACH data GENERATE testcase, COALESCE(val1,100L) as result; - - describe data2; - - data3 = FOREACH data2 GENERATE testcase, result; - - data4 = FOREACH data3 GENERATE testcase, result*100 as result; - - STORE data4 INTO 'output'; - */ - @Multiline private static String coalesceIntAndLongTest2; - - // The first parameter is an int, but the fixed parameter is a long. - // They are merged to a long. - @Test - public void coalesceCastIntToLongTest2() throws Exception - { - PigTest test = createPigTestFromString(coalesceIntAndLongTest2); - - this.writeLinesToFile("input", "1,5", - "2,"); - - test.runScript(); - - List<Tuple> lines = this.getLinesForAlias(test, "data4"); - - Assert.assertEquals(2, lines.size()); - for (Tuple t : lines) - { - switch((Integer)t.get(0)) - { - case 1: - Assert.assertEquals(500L, t.get(1)); break; - case 2: - Assert.assertEquals(10000L, t.get(1)); break; - default: - Assert.fail("Did not expect: " + t.get(1)); - } - } - } - - /** - register $JAR_PATH - - define COALESCE datafu.pig.util.Coalesce('lazy'); - - data = LOAD 'input' using PigStorage(',') AS (testcase:INT,val1:INT); - - data2 = FOREACH data GENERATE testcase, COALESCE(val1,100.0) as result; - - describe data2; - - data3 = FOREACH data2 GENERATE testcase, result; - - data4 = FOREACH data3 GENERATE testcase, result*100 as result; - - STORE data4 INTO 'output'; - */ - @Multiline private static String coalesceIntAndDoubleTest; - - // The first parameter is an int, but the fixed parameter is a long. - // They are merged to a long. - @Test - public void coalesceCastIntToDoubleTest() throws Exception - { - PigTest test = createPigTestFromString(coalesceIntAndDoubleTest); - - this.writeLinesToFile("input", "1,5", - "2,"); - - test.runScript(); - - List<Tuple> lines = this.getLinesForAlias(test, "data4"); - - Assert.assertEquals(2, lines.size()); - for (Tuple t : lines) - { - switch((Integer)t.get(0)) - { - case 1: - Assert.assertEquals(500.0, t.get(1)); break; - case 2: - Assert.assertEquals(10000.0, t.get(1)); break; - default: - Assert.fail("Did not expect: " + t.get(1)); - } - } - } - - /** - register $JAR_PATH - - define COALESCE datafu.pig.util.Coalesce(); - - data = LOAD 'input' using PigStorage(',') AS (testcase:INT,val1:LONG); - - data = FOREACH data GENERATE testcase, (val1 IS NOT NULL ? ToDate(val1) : (datetime)null) as val1; - - data2 = FOREACH data GENERATE testcase, COALESCE(val1,ToDate('1970-01-01T00:00:00.000Z')) as result; - - --describe data2; - - data3 = FOREACH data2 GENERATE testcase, result; - - STORE data3 INTO 'output'; - */ - @Multiline private static String coalesceCastIntToDatetimeTest; - - @Test - public void coalesceCastIntToDatetimeTest() throws Exception - { - PigTest test = createPigTestFromString(coalesceCastIntToDatetimeTest); - - this.writeLinesToFile("input", "1,1375826183000", - "2,"); - - test.runScript(); - - List<Tuple> lines = this.getLinesForAlias(test, "data3"); - - Assert.assertEquals(2, lines.size()); - for (Tuple t : lines) - { - Integer testcase = (Integer)t.get(0); - Assert.assertNotNull(testcase); - switch(testcase) - { - case 1: - Assert.assertEquals("2013-08-06T21:56:23.000Z", ((DateTime)t.get(1)).toDateTime(DateTimeZone.UTC).toString()); break; - case 2: - Assert.assertEquals("1970-01-01T00:00:00.000Z", t.get(1).toString()); break; - default: - Assert.fail("Did not expect: " + t.get(1)); - } - } - } - - /** - register $JAR_PATH - - define COALESCE datafu.pig.util.Coalesce('lazy'); - - data = LOAD 'input' using PigStorage(',') AS (testcase:INT,val1:LONG); - - data = FOREACH data GENERATE testcase, (val1 IS NOT NULL ? ToDate(val1) : (datetime)null) as val1; - - data2 = FOREACH data GENERATE testcase, COALESCE(val1,ToDate('1970-01-01T00:00:00.000Z')) as result; - - --describe data2; - - data3 = FOREACH data2 GENERATE testcase, result; - - STORE data3 INTO 'output'; - */ - @Multiline private static String coalesceCastIntToDatetimeLazyTest; - - @Test - public void coalesceCastIntToDatetimeLazyTest() throws Exception - { - PigTest test = createPigTestFromString(coalesceCastIntToDatetimeLazyTest); - - this.writeLinesToFile("input", "1,1375826183000", - "2,"); - - test.runScript(); - - List<Tuple> lines = this.getLinesForAlias(test, "data3"); - - Assert.assertEquals(2, lines.size()); - for (Tuple t : lines) - { - Integer testcase = (Integer)t.get(0); - Assert.assertNotNull(testcase); - switch(testcase) - { - case 1: - Assert.assertEquals("2013-08-06T21:56:23.000Z", ((DateTime)t.get(1)).toDateTime(DateTimeZone.UTC).toString()); break; - case 2: - Assert.assertEquals("1970-01-01T00:00:00.000Z", t.get(1).toString()); break; - default: - Assert.fail("Did not expect: " + t.get(1)); - } - } - } - - /** - register $JAR_PATH - - define COALESCE datafu.pig.util.Coalesce(); - - data = LOAD 'input' using PigStorage(',') AS (testcase:INT,val1:INT,val2:LONG); - - data2 = FOREACH data GENERATE testcase, COALESCE(val1,val2) as result; - - describe data2; - - data3 = FOREACH data2 GENERATE testcase, result; - - STORE data3 INTO 'output'; - */ - @Multiline private static String coalesceBagIncompatibleTypeTest; - - @Test(expectedExceptions=FrontendException.class) - public void coalesceBagIncompatibleTypeTest() throws Exception - { - PigTest test = createPigTestFromString(coalesceBagIncompatibleTypeTest); - - this.writeLinesToFile("input", "1,1,2L}"); - - test.runScript(); - - this.getLinesForAlias(test, "data3"); - } - - /** - register $JAR_PATH - - define COALESCE datafu.pig.util.Coalesce('lazy'); - define EmptyBagToNullFields datafu.pig.bags.EmptyBagToNullFields(); - - input1 = LOAD 'input1' using PigStorage(',') AS (val1:INT,val2:INT); - input2 = LOAD 'input2' using PigStorage(',') AS (val1:INT,val2:INT); - input3 = LOAD 'input3' using PigStorage(',') AS (val1:INT,val2:INT); - - data4 = COGROUP input1 BY val1, - input2 BY val1, - input3 BY val1; - - dump data4; - - data4 = FOREACH data4 GENERATE - FLATTEN(input1), - FLATTEN(EmptyBagToNullFields(input2)), - FLATTEN(EmptyBagToNullFields(input3)); - - dump data4; - - describe data4; - - data5 = FOREACH data4 GENERATE input1::val1 as val1, COALESCE(input2::val2,0L) as val2, COALESCE(input3::val2,0L) as val3; - - --describe data5; - - STORE data5 INTO 'output'; - */ - @Multiline private static String leftJoinTest; - - @Test - public void leftJoinTest() throws Exception - { - PigTest test = createPigTestFromString(leftJoinTest); - - this.writeLinesToFile("input1", "1,1", - "2,2", - "5,5"); - - this.writeLinesToFile("input2", "1,10", - "3,30", - "5,50"); - - this.writeLinesToFile("input3", "2,100", - "5,500"); - - test.runScript(); - - List<Tuple> lines = this.getLinesForAlias(test, "data5"); - - Assert.assertEquals(3, lines.size()); - for (Tuple t : lines) - { - switch((Integer)t.get(0)) - { - case 1: - Assert.assertEquals(10L, t.get(1)); - Assert.assertEquals(0L, t.get(2)); - break; - case 2: - Assert.assertEquals(0L, t.get(1)); - Assert.assertEquals(100L, t.get(2)); - break; - case 5: - Assert.assertEquals(50L, t.get(1)); - Assert.assertEquals(500L, t.get(2)); - break; - default: - Assert.fail("Did not expect: " + t.get(0)); - } - } - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/test/pig/datafu/test/pig/util/InTests.java ---------------------------------------------------------------------- diff --git a/test/pig/datafu/test/pig/util/InTests.java b/test/pig/datafu/test/pig/util/InTests.java deleted file mode 100644 index a21a0b2..0000000 --- a/test/pig/datafu/test/pig/util/InTests.java +++ /dev/null @@ -1,133 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package datafu.test.pig.util; - -import org.adrianwalker.multilinestring.Multiline; -import org.apache.pig.pigunit.PigTest; -import org.testng.annotations.Test; - -import datafu.test.pig.PigTests; - -public class InTests extends PigTests -{ - /** - register $JAR_PATH - - define I datafu.pig.util.InUDF(); - - data = LOAD 'input' AS (B: bag {T: tuple(v:INT)}); - - data2 = FOREACH data { - C = FILTER B By I(v, 1,2,3); - GENERATE C; - } - - describe data2; - - STORE data2 INTO 'output'; - */ - @Multiline private static String inIntTest; - - @Test - public void inIntTest() throws Exception - { - PigTest test = createPigTestFromString(inIntTest); - - writeLinesToFile("input", - "({(1),(2),(3),(4),(5)})", - "({(1),(2)})", - "({(4),(5)})"); - - test.runScript(); - - assertOutput(test, "data2", - "({(1),(2),(3)})", - "({(1),(2)})", - "({})"); - } - - /** - register $JAR_PATH - - define I datafu.pig.util.InUDF(); - - data = LOAD 'input' AS (B: bag {T: tuple(v:chararray)}); - - data2 = FOREACH data { - C = FILTER B By I(v, 'will','matt','sam'); - GENERATE C; - } - - describe data2; - - STORE data2 INTO 'output'; - */ - @Multiline private static String inStringTest; - - @Test - public void inStringTest() throws Exception - { - PigTest test = createPigTestFromString(inStringTest); - - writeLinesToFile("input", - "({(alice),(bob),(will),(matt),(sam)})", - "({(will),(matt)})", - "({(alice),(bob)})"); - - test.runScript(); - - assertOutput(test, "data2", - "({(will),(matt),(sam)})", - "({(will),(matt)})", - "({})"); - } - - /** - register $JAR_PATH - - define I datafu.pig.util.InUDF(); - - data = LOAD 'input' AS (owner:chararray, color:chararray); - describe data; - data2 = FILTER data BY I(color, 'red','blue'); - describe data2; - STORE data2 INTO 'output'; - */ - @Multiline private static String inTopLevelTest; - - @Test - public void inTopLevelTest() throws Exception - { - PigTest test = createPigTestFromString(inTopLevelTest); - - writeLinesToFile("input", - "alice\tred", - "bob\tblue", - "charlie\tgreen", - "dave\tred"); - test.runScript(); - - assertOutput(test, "data2", - "(alice,red)", - "(bob,blue)", - "(dave,red)"); - } - -} http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/test/pig/datafu/test/pig/util/IntBoolConversionPigTests.java ---------------------------------------------------------------------- diff --git a/test/pig/datafu/test/pig/util/IntBoolConversionPigTests.java b/test/pig/datafu/test/pig/util/IntBoolConversionPigTests.java deleted file mode 100644 index 4017867..0000000 --- a/test/pig/datafu/test/pig/util/IntBoolConversionPigTests.java +++ /dev/null @@ -1,107 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package datafu.test.pig.util; - -import org.adrianwalker.multilinestring.Multiline; -import org.apache.pig.pigunit.PigTest; -import org.testng.annotations.Test; - -import datafu.test.pig.PigTests; - -public class IntBoolConversionPigTests extends PigTests -{ - /** - register $JAR_PATH - - define IntToBool datafu.pig.util.IntToBool(); - - data = LOAD 'input' AS (val:INT); - - data2 = FOREACH data GENERATE IntToBool(val); - - STORE data2 INTO 'output'; - */ - @Multiline private static String intToBoolTest; - - @Test - public void intToBoolTest() throws Exception - { - PigTest test = createPigTestFromString(intToBoolTest); - - String[] input = { - "", // null - "0", - "1" - }; - - String[] output = { - "(false)", - "(false)", - "(true)" - }; - - test.assertOutput("data",input,"data2",output); - } - - /** - register $JAR_PATH - - define IntToBool datafu.pig.util.IntToBool(); - define BoolToInt datafu.pig.util.BoolToInt(); - - data = LOAD 'input' AS (val:INT); - - data2 = FOREACH data GENERATE IntToBool(val) as val; - data3 = FOREACH data2 GENERATE BoolToInt(val) as val; - - STORE data3 INTO 'output'; - */ - @Multiline private static String intToBoolToIntTest; - - @Test - public void intToBoolToIntTest() throws Exception - { - PigTest test = createPigTestFromString(intToBoolToIntTest); - - String[] input = { - "", // null - "0", - "1", - "2", - "-1", - "-2", - "0", - "" - }; - - String[] output = { - "(0)", - "(0)", - "(1)", - "(1)", - "(1)", - "(1)", - "(0)", - "(0)" - }; - - test.assertOutput("data",input,"data3",output); - } -} http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/test/pig/datafu/test/pig/util/TransposeTest.java ---------------------------------------------------------------------- diff --git a/test/pig/datafu/test/pig/util/TransposeTest.java b/test/pig/datafu/test/pig/util/TransposeTest.java deleted file mode 100644 index f131cd0..0000000 --- a/test/pig/datafu/test/pig/util/TransposeTest.java +++ /dev/null @@ -1,116 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package datafu.test.pig.util; - -import java.util.List; - -import org.adrianwalker.multilinestring.Multiline; -import org.apache.pig.data.DataBag; -import org.apache.pig.data.Tuple; -import org.apache.pig.pigunit.PigTest; -import org.testng.Assert; -import org.testng.annotations.Test; - -import datafu.test.pig.PigTests; - -public class TransposeTest extends PigTests -{ - /** - register $JAR_PATH - - define Transpose datafu.pig.util.TransposeTupleToBag(); - - data = LOAD 'input' using PigStorage(',') AS (testcase:INT,val1:INT,val2:INT,val3:INT); - - data2 = FOREACH data GENERATE testcase, Transpose(val1 .. val3) as transposed; - - describe data2; - - data3 = FOREACH data2 GENERATE testcase, transposed; - - STORE data3 INTO 'output'; - */ - @Multiline private static String transposeTest; - - @Test - public void transposeTest() throws Exception - { - PigTest test = createPigTestFromString(transposeTest); - writeLinesToFile("input", "1,10,11,12", - "2,20,21,22"); - test.runScript(); - - List<Tuple> output = getLinesForAlias(test, "data3"); - for (Tuple tuple : output) { - int testCase = (Integer)tuple.get(0); - DataBag bag = (DataBag)tuple.get(1); - Assert.assertEquals(bag.size(), 3); - int i=0; - for (Tuple t : bag) { - String expectedKey = String.format("val%d",i+1); - Assert.assertEquals((String)t.get(0), expectedKey); - int actualValue = (Integer)t.get(1); - Assert.assertEquals(actualValue, testCase*10+i); - i++; - } - } - } - - /** - register $JAR_PATH - - define Transpose datafu.pig.util.TransposeTupleToBag(); - - data = LOAD 'input' using PigStorage(',') AS (testcase:INT,val1:INT,val2:INT,val3:DOUBLE); - - data2 = FOREACH data GENERATE testcase, Transpose(val1 .. val3) as transposed; - - describe data2; - - data3 = FOREACH data2 GENERATE testcase, transposed; - - STORE data3 INTO 'output'; - */ - @Multiline private static String transposeBadTypeTest; - - @Test(expectedExceptions={org.apache.pig.impl.logicalLayer.FrontendException.class}) - public void transposeBadTypeTest() throws Exception - { - PigTest test = createPigTestFromString(transposeBadTypeTest); - writeLinesToFile("input", "1,10,11,12.0", - "2,20,21,22.0"); - test.runScript(); - - List<Tuple> output = getLinesForAlias(test, "data3"); - for (Tuple tuple : output) { - int testCase = (Integer)tuple.get(0); - DataBag bag = (DataBag)tuple.get(1); - Assert.assertEquals(bag.size(), 3); - int i=0; - for (Tuple t : bag) { - String expectedKey = String.format("val%d",i+1); - Assert.assertEquals((String)t.get(0), expectedKey); - int actualValue = (Integer)t.get(1); - Assert.assertEquals(actualValue, testCase*10+i); - i++; - } - } - } -}
