Repository: bigtop Updated Branches: refs/heads/master aaffc1e2c -> 4cee56bdc
http://git-wip-us.apache.org/repos/asf/bigtop/blob/4cee56bd/bigtop-tests/test-artifacts/datafu/src/main/groovy/org/apache/bigtop/itest/datafu/linkanalysis/PageRankTests.java ---------------------------------------------------------------------- diff --git a/bigtop-tests/test-artifacts/datafu/src/main/groovy/org/apache/bigtop/itest/datafu/linkanalysis/PageRankTests.java b/bigtop-tests/test-artifacts/datafu/src/main/groovy/org/apache/bigtop/itest/datafu/linkanalysis/PageRankTests.java deleted file mode 100644 index 11d31bd..0000000 --- a/bigtop-tests/test-artifacts/datafu/src/main/groovy/org/apache/bigtop/itest/datafu/linkanalysis/PageRankTests.java +++ /dev/null @@ -1,120 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * <p/> - * http://www.apache.org/licenses/LICENSE-2.0 - * <p/> - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.bigtop.itest.datafu.linkanalysis; - - -import static org.junit.Assert.*; - -import java.io.BufferedWriter; -import java.io.File; -import java.io.FileWriter; -import java.util.HashMap; -import java.util.Iterator; -import java.util.Map; - -import org.apache.pig.data.Tuple; -import org.apache.pig.pigunit.PigTest; -import org.junit.Test; - - -import org.apache.bigtop.itest.datafu.linkanalysis.PageRankTest; -import org.apache.bigtop.itest.datafu.PigTests; - -public class PageRankTests extends PigTests -{ - @Test - public void pigPageRankTest() throws Exception - { - PigTest test = createPigTest("datafu/linkanalysis/pageRankTest.pig"); - - String[] edges = PageRankTest.getWikiExampleEdges(); - - Map<String,Integer> nodeIds = new HashMap<String,Integer>(); - Map<Integer,String> nodeIdsReversed = new HashMap<Integer,String>(); - Map<String,Float> expectedRanks = PageRankTest.parseExpectedRanks(PageRankTest.getWikiExampleExpectedRanks()); - - File f = new File(System.getProperty("user.dir"), "input").getAbsoluteFile(); - if (f.exists()) - { - f.delete(); - } - - FileWriter writer = new FileWriter(f); - BufferedWriter bufferedWriter = new BufferedWriter(writer); - - for (String edge : edges) - { - String[] edgeParts = edge.split(" "); - String source = edgeParts[0]; - String dest = edgeParts[1]; - if (!nodeIds.containsKey(source)) - { - int id = nodeIds.size(); - nodeIds.put(source,id); - nodeIdsReversed.put(id, source); - } - if (!nodeIds.containsKey(dest)) - { - int id = nodeIds.size(); - nodeIds.put(dest,id); - nodeIdsReversed.put(id, dest); - } - Integer sourceId = nodeIds.get(source); - Integer destId = nodeIds.get(dest); - - StringBuffer sb = new StringBuffer(); - - sb.append("1\t"); // topic - sb.append(sourceId.toString() + "\t"); - sb.append(destId.toString() + "\t"); - sb.append("1.0\n"); // weight - - bufferedWriter.write(sb.toString()); - } - - bufferedWriter.close(); - - test.runScript(); - Iterator<Tuple> tuples = test.getAlias("data_grouped3"); - - System.out.println("Final node ranks:"); - int nodeCount = 0; - while (tuples.hasNext()) - { - Tuple nodeTuple = tuples.next(); - - Integer topic = (Integer)nodeTuple.get(0); - Integer nodeId = (Integer)nodeTuple.get(1); - Float nodeRank = (Float)nodeTuple.get(2); - - assertEquals(1, topic.intValue()); - - System.out.println(String.format("%d => %f", nodeId, nodeRank)); - - Float expectedNodeRank = expectedRanks.get(nodeIdsReversed.get(nodeId)); - - assertTrue(String.format("expected: %f, actual: %f", expectedNodeRank, nodeRank), - Math.abs(expectedNodeRank - nodeRank * 100.0f) < 0.1); - - nodeCount++; - } - - assertEquals(nodeIds.size(),nodeCount); - } -} http://git-wip-us.apache.org/repos/asf/bigtop/blob/4cee56bd/bigtop-tests/test-artifacts/datafu/src/main/groovy/org/apache/bigtop/itest/datafu/numbers/NumberTests.java ---------------------------------------------------------------------- diff --git a/bigtop-tests/test-artifacts/datafu/src/main/groovy/org/apache/bigtop/itest/datafu/numbers/NumberTests.java b/bigtop-tests/test-artifacts/datafu/src/main/groovy/org/apache/bigtop/itest/datafu/numbers/NumberTests.java deleted file mode 100644 index 4408a55..0000000 --- a/bigtop-tests/test-artifacts/datafu/src/main/groovy/org/apache/bigtop/itest/datafu/numbers/NumberTests.java +++ /dev/null @@ -1,65 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * <p/> - * http://www.apache.org/licenses/LICENSE-2.0 - * <p/> - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.bigtop.itest.datafu.numbers; - -import static org.junit.Assert.*; - -import java.util.ArrayList; -import java.util.List; - -import org.apache.pig.data.Tuple; -import org.apache.pig.pigunit.PigTest; -import org.junit.Test; - -import org.apache.bigtop.itest.datafu.PigTests; - -public class NumberTests extends PigTests -{ - /** - * Test the RandomIntRange UDF. The main purpose is to make sure it can be used in a Pig script. - * Also the range of output values is tested. - * - * @throws Exception - */ - @Test - public void randomIntRangeTest() throws Exception - { - PigTest test = createPigTest("datafu/numbers/randomIntRangeTest.pig", - "MIN=1", "MAX=10"); - - List<String> input = new ArrayList<String>(); - for (int i=0; i<100; i++) - { - input.add(String.format("(%d)", i)); - } - - writeLinesToFile("input", - input.toArray(new String[0])); - - test.runScript(); - - List<Tuple> tuples = getLinesForAlias(test, "data2", false); - for (Tuple tuple : tuples) - { - Integer randValue = (Integer)tuple.get(1); - assertTrue(randValue >= 1); - assertTrue(randValue <= 10); - } - } -} http://git-wip-us.apache.org/repos/asf/bigtop/blob/4cee56bd/bigtop-tests/test-artifacts/datafu/src/main/groovy/org/apache/bigtop/itest/datafu/sessions/SessionTests.java ---------------------------------------------------------------------- diff --git a/bigtop-tests/test-artifacts/datafu/src/main/groovy/org/apache/bigtop/itest/datafu/sessions/SessionTests.java b/bigtop-tests/test-artifacts/datafu/src/main/groovy/org/apache/bigtop/itest/datafu/sessions/SessionTests.java deleted file mode 100644 index d13f1c3..0000000 --- a/bigtop-tests/test-artifacts/datafu/src/main/groovy/org/apache/bigtop/itest/datafu/sessions/SessionTests.java +++ /dev/null @@ -1,92 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * <p/> - * http://www.apache.org/licenses/LICENSE-2.0 - * <p/> - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.bigtop.itest.datafu.sessions; - -import static org.junit.Assert.*; - -import java.util.HashMap; - -import org.apache.pig.data.Tuple; -import org.apache.pig.pigunit.PigTest; -import org.junit.Test; - -import org.apache.bigtop.itest.datafu.PigTests; - -public class SessionTests extends PigTests -{ - @Test - public void sessionizeTest() throws Exception - { - PigTest test = createPigTest("datafu/sessions/sessionizeTest.pig", - "TIME_WINDOW=30m", - "JAR_PATH=" + getJarPath()); - - this.writeLinesToFile("input", - "2010-01-01T01:00:00Z\t1\t10", - "2010-01-01T01:15:00Z\t1\t20", - "2010-01-01T01:31:00Z\t1\t10", - "2010-01-01T01:35:00Z\t1\t20", - "2010-01-01T02:30:00Z\t1\t30", - - "2010-01-01T01:00:00Z\t2\t10", - "2010-01-01T01:31:00Z\t2\t20", - "2010-01-01T02:10:00Z\t2\t30", - "2010-01-01T02:40:30Z\t2\t40", - "2010-01-01T03:30:00Z\t2\t50", - - "2010-01-01T01:00:00Z\t3\t10", - "2010-01-01T01:01:00Z\t3\t20", - "2010-01-01T01:02:00Z\t3\t5", - "2010-01-01T01:10:00Z\t3\t25", - "2010-01-01T01:15:00Z\t3\t50", - "2010-01-01T01:25:00Z\t3\t30", - "2010-01-01T01:30:00Z\t3\t15"); - - test.runScript(); - - HashMap<Integer,HashMap<Integer,Boolean>> userValues = new HashMap<Integer,HashMap<Integer,Boolean>>(); - - for (Tuple t : this.getLinesForAlias(test, "max_value")) - { - Integer userId = (Integer)t.get(0); - Integer max = (Integer)t.get(1); - if (!userValues.containsKey(userId)) - { - userValues.put(userId, new HashMap<Integer,Boolean>()); - } - userValues.get(userId).put(max, true); - } - - assertEquals(2, userValues.get(1).size()); - assertEquals(5, userValues.get(2).size()); - assertEquals(1, userValues.get(3).size()); - - assertTrue(userValues.get(1).containsKey(20)); - assertTrue(userValues.get(1).containsKey(30)); - - assertTrue(userValues.get(2).containsKey(10)); - assertTrue(userValues.get(2).containsKey(20)); - assertTrue(userValues.get(2).containsKey(30)); - assertTrue(userValues.get(2).containsKey(40)); - assertTrue(userValues.get(2).containsKey(50)); - - assertTrue(userValues.get(3).containsKey(50)); - } -} - http://git-wip-us.apache.org/repos/asf/bigtop/blob/4cee56bd/bigtop-tests/test-artifacts/datafu/src/main/groovy/org/apache/bigtop/itest/datafu/stats/MarkovPairTests.java ---------------------------------------------------------------------- diff --git a/bigtop-tests/test-artifacts/datafu/src/main/groovy/org/apache/bigtop/itest/datafu/stats/MarkovPairTests.java b/bigtop-tests/test-artifacts/datafu/src/main/groovy/org/apache/bigtop/itest/datafu/stats/MarkovPairTests.java deleted file mode 100644 index f1f1c2f..0000000 --- a/bigtop-tests/test-artifacts/datafu/src/main/groovy/org/apache/bigtop/itest/datafu/stats/MarkovPairTests.java +++ /dev/null @@ -1,105 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * <p/> - * http://www.apache.org/licenses/LICENSE-2.0 - * <p/> - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.bigtop.itest.datafu.stats; - -import static org.junit.Assert.*; - -import java.util.Iterator; - -import org.apache.pig.data.Tuple; -import org.apache.pig.pigunit.PigTest; -import org.junit.Test; - -import org.apache.bigtop.itest.datafu.PigTests; - -public class MarkovPairTests extends PigTests -{ - @Test - public void markovPairDefaultTest() throws Exception - { - PigTest test = createPigTest("datafu/stats/markovPairDefault.pig", - "schema=(data: bag {t: tuple(val:int)})"); - - writeLinesToFile("input", "{(10),(20),(30),(40),(50),(60)}"); - - String[] expectedOutput = { - "({((10),(20)),((20),(30)),((30),(40)),((40),(50)),((50),(60))})" - }; - - test.runScript(); - - Iterator<Tuple> actualOutput = test.getAlias("data_out"); - - assertTuplesMatch(expectedOutput, actualOutput); - } - - @Test - public void markovPairMultipleInput() throws Exception - { - PigTest test = createPigTest("datafu/stats/markovPairDefault.pig", - "schema=(data: bag {t: tuple(val1:int,val2:int)})"); - - writeLinesToFile("input", "{(10,100),(20,200),(30,300),(40,400),(50,500),(60,600)}"); - - String[] expectedOutput = { - "({((10,100),(20,200)),((20,200),(30,300)),((30,300),(40,400)),((40,400),(50,500)),((50,500),(60,600))})" - }; - - - test.runScript(); - - Iterator<Tuple> actualOutput = test.getAlias("data_out"); - - assertTuplesMatch(expectedOutput, actualOutput); - } - - @Test - public void markovPairLookaheadTest() throws Exception - { - PigTest test = createPigTest("datafu/stats/markovPairLookahead.pig", - "schema=(data: bag {t: tuple(val:int)})", - "lookahead=3"); - - writeLinesToFile("input", "{(10),(20),(30),(40),(50)}"); - - String[] expectedOutput = { - "({((10),(20)),((10),(30)),((10),(40)),((20),(30)),((20),(40)),((20),(50)),((30),(40)),((30),(50)),((40),(50))})" - }; - - test.runScript(); - - Iterator<Tuple> actualOutput = test.getAlias("data_out"); - - assertTuplesMatch(expectedOutput, actualOutput); - } - - private void assertTuplesMatch(String[] expectedOutput, Iterator<Tuple> actualOutput) - { - Iterator<Tuple> tuples = actualOutput; - - for (String outputLine : expectedOutput) - { - assertTrue(tuples.hasNext()); - Tuple outputTuple = tuples.next(); - System.out.println(String.format("expected: %s", outputLine)); - System.out.println(String.format("actual: %s", outputTuple.toString())); - assertEquals(outputLine,outputTuple.toString()); - } - } -} http://git-wip-us.apache.org/repos/asf/bigtop/blob/4cee56bd/bigtop-tests/test-artifacts/datafu/src/main/groovy/org/apache/bigtop/itest/datafu/stats/QuantileTests.java ---------------------------------------------------------------------- diff --git a/bigtop-tests/test-artifacts/datafu/src/main/groovy/org/apache/bigtop/itest/datafu/stats/QuantileTests.java b/bigtop-tests/test-artifacts/datafu/src/main/groovy/org/apache/bigtop/itest/datafu/stats/QuantileTests.java deleted file mode 100644 index e9ef05d..0000000 --- a/bigtop-tests/test-artifacts/datafu/src/main/groovy/org/apache/bigtop/itest/datafu/stats/QuantileTests.java +++ /dev/null @@ -1,196 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * <p/> - * http://www.apache.org/licenses/LICENSE-2.0 - * <p/> - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.bigtop.itest.datafu.stats; - -import static org.junit.Assert.*; - -import java.util.ArrayList; -import java.util.List; - -import org.apache.pig.data.Tuple; -import org.apache.pig.pigunit.PigTest; -import org.junit.Test; - -import org.apache.bigtop.itest.datafu.PigTests; - -public class QuantileTests extends PigTests -{ - @Test - public void quantileTest() throws Exception - { - PigTest test = createPigTest("datafu/stats/quantileTest.pig", - "QUANTILES='0.0','0.25','0.5','0.75','1.0'"); - - String[] input = {"1","2","3","4","10","5","6","7","8","9"}; - writeLinesToFile("input", input); - - test.runScript(); - - List<Tuple> output = getLinesForAlias(test, "data_out", true); - - assertEquals(1,output.size()); - assertEquals("(1.0,3.0,5.5,8.0,10.0)", output.get(0).toString()); - } - - @Test - public void quantile2Test() throws Exception - { - PigTest test = createPigTest("datafu/stats/quantileTest.pig", - "QUANTILES='5'"); - - String[] input = {"1","2","3","4","10","5","6","7","8","9"}; - writeLinesToFile("input", input); - - test.runScript(); - - List<Tuple> output = getLinesForAlias(test, "data_out", true); - - assertEquals(1,output.size()); - assertEquals("(1.0,3.0,5.5,8.0,10.0)", output.get(0).toString()); - } - - @Test - public void medianTest() throws Exception - { - PigTest test = createPigTest("datafu/stats/medianTest.pig"); - - String[] input = {"4","5","6","9","10","7","8","2","3","1"}; - writeLinesToFile("input", input); - - test.runScript(); - - List<Tuple> output = getLinesForAlias(test, "data_out", true); - - assertEquals(1,output.size()); - assertEquals("(5.5)", output.get(0).toString()); - } - - @Test - public void streamingMedianTest() throws Exception - { - PigTest test = createPigTest("datafu/stats/streamingMedianTest.pig"); - - String[] input = {"0","4","5","6","9","10","7","8","2","3","1"}; - writeLinesToFile("input", input); - - test.runScript(); - - List<Tuple> output = getLinesForAlias(test, "data_out", true); - - assertEquals(1,output.size()); - assertEquals("(5.0)", output.get(0).toString()); - } - - @Test - public void streamingQuantileTest() throws Exception { - PigTest test = createPigTest("datafu/stats/streamingQuantileTest.pig", - "QUANTILES='5'"); - - String[] input = {"1","2","3","4","10","5","6","7","8","9"}; - writeLinesToFile("input", input); - - test.runScript(); - - List<Tuple> output = getLinesForAlias(test, "data_out", true); - - assertEquals(1,output.size()); - assertEquals("(1.0,3.0,5.0,8.0,10.0)", output.get(0).toString()); - } - - @Test - public void streamingQuantile2Test() throws Exception { - PigTest test = createPigTest("datafu/stats/streamingQuantileTest.pig", - "QUANTILES='0.5','0.75','1.0'"); - - String[] input = {"1","2","3","4","10","5","6","7","8","9"}; - writeLinesToFile("input", input); - - test.runScript(); - - List<Tuple> output = getLinesForAlias(test, "data_out", true); - - assertEquals(1,output.size()); - assertEquals("(5.0,8.0,10.0)", output.get(0).toString()); - } - - @Test - public void streamingQuantile3Test() throws Exception { - PigTest test = createPigTest("datafu/stats/streamingQuantileTest.pig", - "QUANTILES='0.07','0.03','0.37','1.0','0.0'"); - - List<String> input = new ArrayList<String>(); - for (int i=1000; i>=1; i--) - { - input.add(Integer.toString(i)); - } - - writeLinesToFile("input", input.toArray(new String[0])); - - test.runScript(); - - List<Tuple> output = getLinesForAlias(test, "data_out", true); - - assertEquals(1,output.size()); - assertEquals("(70.0,30.0,370.0,1000.0,1.0)", output.get(0).toString()); - } - - @Test - public void streamingQuantile4Test() throws Exception { - PigTest test = createPigTest("datafu/stats/streamingQuantileTest.pig", - "QUANTILES='0.0013','0.0228','0.1587','0.5','0.8413','0.9772','0.9987'"); - - List<String> input = new ArrayList<String>(); - for (int i=100000; i>=0; i--) - { - input.add(Integer.toString(i)); - } - - writeLinesToFile("input", input.toArray(new String[0])); - - test.runScript(); - - List<Tuple> output = getLinesForAlias(test, "data_out", true); - - assertEquals(1,output.size()); - assertEquals("(130.0,2280.0,15870.0,50000.0,84130.0,97720.0,99870.0)", output.get(0).toString()); - } - - - - @Test - public void quantile3Test() throws Exception { - PigTest test = createPigTest("datafu/stats/quantileTest.pig", - "QUANTILES='0.0013','0.0228','0.1587','0.5','0.8413','0.9772','0.9987'"); - - List<String> input = new ArrayList<String>(); - for (int i=100000; i>=0; i--) - { - input.add(Integer.toString(i)); - } - - writeLinesToFile("input", input.toArray(new String[0])); - - test.runScript(); - - List<Tuple> output = getLinesForAlias(test, "data_out", true); - - assertEquals(1,output.size()); - assertEquals("(130.0,2280.0,15870.0,50000.0,84130.0,97720.0,99870.0)", output.get(0).toString()); - } -} http://git-wip-us.apache.org/repos/asf/bigtop/blob/4cee56bd/bigtop-tests/test-artifacts/datafu/src/main/groovy/org/apache/bigtop/itest/datafu/stats/WilsonBinConfTests.java ---------------------------------------------------------------------- diff --git a/bigtop-tests/test-artifacts/datafu/src/main/groovy/org/apache/bigtop/itest/datafu/stats/WilsonBinConfTests.java b/bigtop-tests/test-artifacts/datafu/src/main/groovy/org/apache/bigtop/itest/datafu/stats/WilsonBinConfTests.java deleted file mode 100644 index cb43ce1..0000000 --- a/bigtop-tests/test-artifacts/datafu/src/main/groovy/org/apache/bigtop/itest/datafu/stats/WilsonBinConfTests.java +++ /dev/null @@ -1,81 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * <p/> - * http://www.apache.org/licenses/LICENSE-2.0 - * <p/> - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.bigtop.itest.datafu.stats; - -import static org.junit.Assert.*; - -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; - -import org.apache.pig.data.Tuple; -import org.apache.pig.pigunit.PigTest; -import org.junit.Test; - -import org.apache.bigtop.itest.datafu.PigTests; - -public class WilsonBinConfTests extends PigTests -{ - @Test - public void wilsonTest() throws Exception - { - PigTest test = createPigTest("datafu/stats/wilsonBinConfTests.pig", - "alpha=0.05"); // alpha is 0.05 for 95% confidence - - writeLinesToFile("input", - "1\t1", - "1\t2", - "50\t100", - "500\t1000", - "999\t1000", - "1000\t1000", - "998\t1000"); - - test.runScript(); - - /* Add expected values, computed using R: - * - * e.g. - * - * library(Hmisc) - * - * binconf(50,100) - * binconf(500,1000) - * - */ - List<String> expectedOutput = new ArrayList<String>(); - expectedOutput.add("0.05129,1.00000"); - expectedOutput.add("0.02565,0.97435"); - expectedOutput.add("0.40383,0.59617"); - expectedOutput.add("0.46907,0.53093"); - expectedOutput.add("0.99436,0.99995"); - expectedOutput.add("0.99617,1.00000"); - expectedOutput.add("0.99274,0.99945"); - - List<Tuple> output = this.getLinesForAlias(test, "data_out"); - Iterator<String> expectationIterator = expectedOutput.iterator(); - for (Tuple t : output) - { - assertTrue(expectationIterator.hasNext()); - Double lower = (Double)t.get(0); - Double upper = (Double)t.get(1); - assertEquals(expectationIterator.next(),String.format("%.5f,%.5f",lower,upper)); - } - } -} http://git-wip-us.apache.org/repos/asf/bigtop/blob/4cee56bd/bigtop-tests/test-artifacts/datafu/src/main/groovy/org/apache/bigtop/itest/datafu/urls/UserAgentTest.java ---------------------------------------------------------------------- diff --git a/bigtop-tests/test-artifacts/datafu/src/main/groovy/org/apache/bigtop/itest/datafu/urls/UserAgentTest.java b/bigtop-tests/test-artifacts/datafu/src/main/groovy/org/apache/bigtop/itest/datafu/urls/UserAgentTest.java deleted file mode 100644 index e742c0d..0000000 --- a/bigtop-tests/test-artifacts/datafu/src/main/groovy/org/apache/bigtop/itest/datafu/urls/UserAgentTest.java +++ /dev/null @@ -1,57 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * <p/> - * http://www.apache.org/licenses/LICENSE-2.0 - * <p/> - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.bigtop.itest.datafu.urls; - -import org.apache.pig.pigunit.PigTest; -import org.junit.Test; - -import org.apache.bigtop.itest.datafu.PigTests; - -public class UserAgentTest extends PigTests -{ - - @Test - public void userAgentTest() throws Exception - { - PigTest test = createPigTest("datafu/urls/userAgentTest.pig"); - - String[] input = { - "Mozilla/5.0 (iPhone; U; CPU iPhone OS 4_3_3 like Mac OS X; en-us) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8J2 Safari/6533.18.5", - "Mozilla/5.0 (compatible; Konqueror/3.5; Linux; X11; de) KHTML/3.5.2 (like Gecko) Kubuntu 6.06 Dapper", - "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:2.2a1pre) Gecko/20110331 Firefox/4.2a1pre Fennec/4.1a1pre", - "Opera/9.00 (X11; Linux i686; U; en)", - "Wget/1.10.2", - "Opera/9.80 (Android; Linux; Opera Mobi/ADR-1012221546; U; pl) Presto/2.7.60 Version/10.5", - "Mozilla/5.0 (Linux; U; Android 2.2; en-us; DROID2 Build/VZW) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1" - }; - - String[] output = { - "(mobile)", - "(desktop)", - "(mobile)", - "(desktop)", - "(desktop)", - "(mobile)", - "(mobile)", - }; - - test.assertOutput("data",input,"data_out",output); - } - -} http://git-wip-us.apache.org/repos/asf/bigtop/blob/4cee56bd/bigtop-tests/test-artifacts/datafu/src/main/groovy/org/apache/bigtop/itest/datafu/util/AssertTests.java ---------------------------------------------------------------------- diff --git a/bigtop-tests/test-artifacts/datafu/src/main/groovy/org/apache/bigtop/itest/datafu/util/AssertTests.java b/bigtop-tests/test-artifacts/datafu/src/main/groovy/org/apache/bigtop/itest/datafu/util/AssertTests.java deleted file mode 100644 index 0379ae7..0000000 --- a/bigtop-tests/test-artifacts/datafu/src/main/groovy/org/apache/bigtop/itest/datafu/util/AssertTests.java +++ /dev/null @@ -1,93 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * <p/> - * http://www.apache.org/licenses/LICENSE-2.0 - * <p/> - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.bigtop.itest.datafu.util; - -import static org.junit.Assert.*; - -import org.apache.pig.pigunit.PigTest; -import org.junit.Test; - -import org.apache.bigtop.itest.datafu.PigTests; - -public class AssertTests extends PigTests -{ - @Test - public void shouldAssertWithMessageOnZero() throws Exception - { - try - { - PigTest test = createPigTest("datafu/util/assertWithMessageTest.pig"); - - this.writeLinesToFile("input", "0"); - - test.runScript(); - - this.getLinesForAlias(test, "data2"); - - fail("test should have failed, but it didn't"); - } - catch (Exception e) - { - } - } - - @Test - public void shouldNotAssertWithMessageOnOne() throws Exception - { - PigTest test = createPigTest("datafu/util/assertWithMessageTest.pig"); - - this.writeLinesToFile("input", "1"); - - test.runScript(); - - this.getLinesForAlias(test, "data2"); - } - - @Test - public void shouldAssertWithoutMessageOnZero() throws Exception - { - try - { - PigTest test = createPigTest("datafu/util/assertWithoutMessageTest.pig"); - - this.writeLinesToFile("input", "0"); - - test.runScript(); - - this.getLinesForAlias(test, "data2"); - - fail("test should have failed, but it didn't"); - } - catch (Exception e) - { - } - } - - @Test - public void shouldNotAssertWithoutMessageOnOne() throws Exception - { - PigTest test = createPigTest("datafu/util/assertWithoutMessageTest.pig"); - - this.writeLinesToFile("input", "1"); - - test.runScript(); - - this.getLinesForAlias(test, "data2"); - } -} http://git-wip-us.apache.org/repos/asf/bigtop/blob/4cee56bd/bigtop-tests/test-artifacts/datafu/src/main/groovy/org/apache/bigtop/itest/datafu/util/IntBoolConversionPigTests.java ---------------------------------------------------------------------- diff --git a/bigtop-tests/test-artifacts/datafu/src/main/groovy/org/apache/bigtop/itest/datafu/util/IntBoolConversionPigTests.java b/bigtop-tests/test-artifacts/datafu/src/main/groovy/org/apache/bigtop/itest/datafu/util/IntBoolConversionPigTests.java deleted file mode 100644 index 2653060..0000000 --- a/bigtop-tests/test-artifacts/datafu/src/main/groovy/org/apache/bigtop/itest/datafu/util/IntBoolConversionPigTests.java +++ /dev/null @@ -1,77 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * <p/> - * http://www.apache.org/licenses/LICENSE-2.0 - * <p/> - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.bigtop.itest.datafu.util; - -import org.apache.pig.pigunit.PigTest; -import org.junit.Test; - -import org.apache.bigtop.itest.datafu.PigTests; - -public class IntBoolConversionPigTests extends PigTests -{ - @Test - public void intToBoolTest() throws Exception - { - PigTest test = createPigTest("datafu/util/intToBoolTest.pig"); - - String[] input = { - "", // null - "0", - "1" - }; - - String[] output = { - "(false)", - "(false)", - "(true)" - }; - - test.assertOutput("data",input,"data2",output); - } - - @Test - public void intToBoolToIntTest() throws Exception - { - PigTest test = createPigTest("datafu/util/intToBoolToIntTest.pig"); - - String[] input = { - "", // null - "0", - "1", - "2", - "-1", - "-2", - "0", - "" - }; - - String[] output = { - "(0)", - "(0)", - "(1)", - "(1)", - "(1)", - "(1)", - "(0)", - "(0)" - }; - - test.assertOutput("data",input,"data3",output); - } -} http://git-wip-us.apache.org/repos/asf/bigtop/blob/4cee56bd/bigtop-tests/test-artifacts/datafu/src/main/resources/datafu/bags/aliasBagFieldsTest.pig ---------------------------------------------------------------------- diff --git a/bigtop-tests/test-artifacts/datafu/src/main/resources/datafu/bags/aliasBagFieldsTest.pig b/bigtop-tests/test-artifacts/datafu/src/main/resources/datafu/bags/aliasBagFieldsTest.pig deleted file mode 100644 index 247c832..0000000 --- a/bigtop-tests/test-artifacts/datafu/src/main/resources/datafu/bags/aliasBagFieldsTest.pig +++ /dev/null @@ -1,20 +0,0 @@ -register $JAR_PATH - -define AliasBagFields datafu.pig.bags.AliasBagFields('[a#alpha,b#numeric]'); - -data = LOAD 'input' AS (data: bag {T: tuple(a:CHARARRAY, b:INT, c:INT)}); - -data2 = FOREACH data GENERATE AliasBagFields(data) as data; - -describe data2; - -data3 = FOREACH data2 GENERATE FLATTEN(data); - -describe data3; - -data4 = FOREACH data3 GENERATE data::alpha, data::numeric; - -describe data4; - -STORE data4 INTO 'output'; - http://git-wip-us.apache.org/repos/asf/bigtop/blob/4cee56bd/bigtop-tests/test-artifacts/datafu/src/main/resources/datafu/bags/appendToBagTest.pig ---------------------------------------------------------------------- diff --git a/bigtop-tests/test-artifacts/datafu/src/main/resources/datafu/bags/appendToBagTest.pig b/bigtop-tests/test-artifacts/datafu/src/main/resources/datafu/bags/appendToBagTest.pig deleted file mode 100644 index d906bc4..0000000 --- a/bigtop-tests/test-artifacts/datafu/src/main/resources/datafu/bags/appendToBagTest.pig +++ /dev/null @@ -1,9 +0,0 @@ -register $JAR_PATH - -define AppendToBag datafu.pig.bags.AppendToBag(); - -data = LOAD 'input' AS (key:INT, B: bag{T: tuple(v:INT)}, T: tuple(v:INT)); - -data2 = FOREACH data GENERATE key, AppendToBag(B,T) as B; - -STORE data2 INTO 'output'; http://git-wip-us.apache.org/repos/asf/bigtop/blob/4cee56bd/bigtop-tests/test-artifacts/datafu/src/main/resources/datafu/bags/bagConcatTest.pig ---------------------------------------------------------------------- diff --git a/bigtop-tests/test-artifacts/datafu/src/main/resources/datafu/bags/bagConcatTest.pig b/bigtop-tests/test-artifacts/datafu/src/main/resources/datafu/bags/bagConcatTest.pig deleted file mode 100644 index 30d46a0..0000000 --- a/bigtop-tests/test-artifacts/datafu/src/main/resources/datafu/bags/bagConcatTest.pig +++ /dev/null @@ -1,11 +0,0 @@ -register $JAR_PATH - -define BagConcat datafu.pig.bags.BagConcat(); - -data = LOAD 'input' AS (A: bag{T: tuple(v:INT)}, B: bag{T: tuple(v:INT)}, C: bag{T: tuple(v:INT)}); - -data2 = FOREACH data GENERATE BagConcat(A,B,C); - -describe data2 - -STORE data2 INTO 'output'; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/bigtop/blob/4cee56bd/bigtop-tests/test-artifacts/datafu/src/main/resources/datafu/bags/bagSplitTest.pig ---------------------------------------------------------------------- diff --git a/bigtop-tests/test-artifacts/datafu/src/main/resources/datafu/bags/bagSplitTest.pig b/bigtop-tests/test-artifacts/datafu/src/main/resources/datafu/bags/bagSplitTest.pig deleted file mode 100644 index ee4f538..0000000 --- a/bigtop-tests/test-artifacts/datafu/src/main/resources/datafu/bags/bagSplitTest.pig +++ /dev/null @@ -1,14 +0,0 @@ -register $JAR_PATH - -define BagSplit datafu.pig.bags.BagSplit(); - -data = LOAD 'input' AS (B:bag{T:tuple(val1:INT,val2:INT)}); - -data2 = FOREACH data GENERATE BagSplit($MAX,B); -describe data2; - -data3 = FOREACH data2 GENERATE FLATTEN($0); - -describe data3 - -STORE data3 INTO 'output'; http://git-wip-us.apache.org/repos/asf/bigtop/blob/4cee56bd/bigtop-tests/test-artifacts/datafu/src/main/resources/datafu/bags/bagSplitWithBagNumTest.pig ---------------------------------------------------------------------- diff --git a/bigtop-tests/test-artifacts/datafu/src/main/resources/datafu/bags/bagSplitWithBagNumTest.pig b/bigtop-tests/test-artifacts/datafu/src/main/resources/datafu/bags/bagSplitWithBagNumTest.pig deleted file mode 100644 index 833e912..0000000 --- a/bigtop-tests/test-artifacts/datafu/src/main/resources/datafu/bags/bagSplitWithBagNumTest.pig +++ /dev/null @@ -1,11 +0,0 @@ -register $JAR_PATH - -define BagSplit datafu.pig.bags.BagSplit('true'); - -data = LOAD 'input' AS (B:bag{T:tuple(val1:INT,val2:INT)}); - -data2 = FOREACH data GENERATE BagSplit($MAX,B); - -data3 = FOREACH data2 GENERATE FLATTEN($0); - -STORE data3 INTO 'output'; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/bigtop/blob/4cee56bd/bigtop-tests/test-artifacts/datafu/src/main/resources/datafu/bags/comprehensiveBagSplitAndEnumerate.pig ---------------------------------------------------------------------- diff --git a/bigtop-tests/test-artifacts/datafu/src/main/resources/datafu/bags/comprehensiveBagSplitAndEnumerate.pig b/bigtop-tests/test-artifacts/datafu/src/main/resources/datafu/bags/comprehensiveBagSplitAndEnumerate.pig deleted file mode 100644 index 88d7392..0000000 --- a/bigtop-tests/test-artifacts/datafu/src/main/resources/datafu/bags/comprehensiveBagSplitAndEnumerate.pig +++ /dev/null @@ -1,26 +0,0 @@ -register $JAR_PATH - -define BagSplit datafu.pig.bags.BagSplit(); -define Enumerate datafu.pig.bags.Enumerate('1'); - -data = LOAD 'input' AS (data: bag {T: tuple(name:CHARARRAY, score:double)}); - -data2 = FOREACH data GENERATE BagSplit(3,data) as the_bags; - -describe data2 - -data3 = FOREACH data2 GENERATE Enumerate(the_bags) as enumerated_bags; - -describe data3 - -data4 = FOREACH data3 GENERATE FLATTEN(enumerated_bags) as (data,i); - -describe data4 - -data5 = FOREACH data4 GENERATE data as the_data, i as the_key; - -describe data5 - -data_out = FOREACH data5 GENERATE FLATTEN(the_data), the_key; - -describe data_out \ No newline at end of file http://git-wip-us.apache.org/repos/asf/bigtop/blob/4cee56bd/bigtop-tests/test-artifacts/datafu/src/main/resources/datafu/bags/distinctByTest.pig ---------------------------------------------------------------------- diff --git a/bigtop-tests/test-artifacts/datafu/src/main/resources/datafu/bags/distinctByTest.pig b/bigtop-tests/test-artifacts/datafu/src/main/resources/datafu/bags/distinctByTest.pig deleted file mode 100644 index 9532d07..0000000 --- a/bigtop-tests/test-artifacts/datafu/src/main/resources/datafu/bags/distinctByTest.pig +++ /dev/null @@ -1,12 +0,0 @@ -register $JAR_PATH - -define DistinctBy datafu.pig.bags.DistinctBy('0'); - -data = LOAD 'input' AS (data: bag {T: tuple(a:CHARARRAY, b:INT, c:INT)}); - -data2 = FOREACH data GENERATE DistinctBy(data); - -describe data2; - -STORE data2 INTO 'output'; - http://git-wip-us.apache.org/repos/asf/bigtop/blob/4cee56bd/bigtop-tests/test-artifacts/datafu/src/main/resources/datafu/bags/enumerateTest.pig ---------------------------------------------------------------------- diff --git a/bigtop-tests/test-artifacts/datafu/src/main/resources/datafu/bags/enumerateTest.pig b/bigtop-tests/test-artifacts/datafu/src/main/resources/datafu/bags/enumerateTest.pig deleted file mode 100644 index 1647485..0000000 --- a/bigtop-tests/test-artifacts/datafu/src/main/resources/datafu/bags/enumerateTest.pig +++ /dev/null @@ -1,16 +0,0 @@ -register $JAR_PATH - -define Enumerate datafu.pig.bags.Enumerate(); - -data = LOAD 'input' AS (data: bag {T: tuple(v1:INT,B: bag{T: tuple(v2:INT)})}); - -data2 = FOREACH data GENERATE Enumerate(data); -describe data2; - -data3 = FOREACH data2 GENERATE FLATTEN($0); -describe data3; - -data4 = FOREACH data3 GENERATE $0 as v1, $1 as B, $2 as i; -describe data4; - -STORE data4 INTO 'output'; http://git-wip-us.apache.org/repos/asf/bigtop/blob/4cee56bd/bigtop-tests/test-artifacts/datafu/src/main/resources/datafu/bags/enumerateWithReverseTest.pig ---------------------------------------------------------------------- diff --git a/bigtop-tests/test-artifacts/datafu/src/main/resources/datafu/bags/enumerateWithReverseTest.pig b/bigtop-tests/test-artifacts/datafu/src/main/resources/datafu/bags/enumerateWithReverseTest.pig deleted file mode 100644 index 1f04b04..0000000 --- a/bigtop-tests/test-artifacts/datafu/src/main/resources/datafu/bags/enumerateWithReverseTest.pig +++ /dev/null @@ -1,16 +0,0 @@ -register $JAR_PATH - -define Enumerate datafu.pig.bags.Enumerate('1', 'true'); - -data = LOAD 'input' AS (data: bag {T: tuple(v1:INT,B: bag{T: tuple(v2:INT)})}); - -data2 = FOREACH data GENERATE Enumerate(data); -describe data2; - -data3 = FOREACH data2 GENERATE FLATTEN($0); -describe data3; - -data4 = FOREACH data3 GENERATE $0 as v1, $1 as B, $2 as i; -describe data4; - -STORE data4 INTO 'output'; http://git-wip-us.apache.org/repos/asf/bigtop/blob/4cee56bd/bigtop-tests/test-artifacts/datafu/src/main/resources/datafu/bags/enumerateWithStartTest.pig ---------------------------------------------------------------------- diff --git a/bigtop-tests/test-artifacts/datafu/src/main/resources/datafu/bags/enumerateWithStartTest.pig b/bigtop-tests/test-artifacts/datafu/src/main/resources/datafu/bags/enumerateWithStartTest.pig deleted file mode 100644 index d288a6e..0000000 --- a/bigtop-tests/test-artifacts/datafu/src/main/resources/datafu/bags/enumerateWithStartTest.pig +++ /dev/null @@ -1,16 +0,0 @@ -register $JAR_PATH - -define Enumerate datafu.pig.bags.Enumerate('1'); - -data = LOAD 'input' AS (data: bag {T: tuple(v1:INT,B: bag{T: tuple(v2:INT)})}); - -data2 = FOREACH data GENERATE Enumerate(data); -describe data2; - -data3 = FOREACH data2 GENERATE FLATTEN($0); -describe data3; - -data4 = FOREACH data3 GENERATE $0 as v1, $1 as B, $2 as i; -describe data4; - -STORE data4 INTO 'output'; http://git-wip-us.apache.org/repos/asf/bigtop/blob/4cee56bd/bigtop-tests/test-artifacts/datafu/src/main/resources/datafu/bags/firstTupleFromBagTest.pig ---------------------------------------------------------------------- diff --git a/bigtop-tests/test-artifacts/datafu/src/main/resources/datafu/bags/firstTupleFromBagTest.pig b/bigtop-tests/test-artifacts/datafu/src/main/resources/datafu/bags/firstTupleFromBagTest.pig deleted file mode 100644 index 921787e..0000000 --- a/bigtop-tests/test-artifacts/datafu/src/main/resources/datafu/bags/firstTupleFromBagTest.pig +++ /dev/null @@ -1,9 +0,0 @@ -register $JAR_PATH - -define FirstTupleFromBag datafu.pig.bags.FirstTupleFromBag(); - -data = LOAD 'input' AS (key:INT, B: bag{T: tuple(v:INT)}); - -data2 = FOREACH data GENERATE key, FirstTupleFromBag(B, null) as B; - -STORE data2 INTO 'output'; http://git-wip-us.apache.org/repos/asf/bigtop/blob/4cee56bd/bigtop-tests/test-artifacts/datafu/src/main/resources/datafu/bags/nullToEmptyBagTest.pig ---------------------------------------------------------------------- diff --git a/bigtop-tests/test-artifacts/datafu/src/main/resources/datafu/bags/nullToEmptyBagTest.pig b/bigtop-tests/test-artifacts/datafu/src/main/resources/datafu/bags/nullToEmptyBagTest.pig deleted file mode 100644 index 3e809b3..0000000 --- a/bigtop-tests/test-artifacts/datafu/src/main/resources/datafu/bags/nullToEmptyBagTest.pig +++ /dev/null @@ -1,14 +0,0 @@ -register $JAR_PATH - -define NullToEmptyBag datafu.pig.bags.NullToEmptyBag(); - -data = LOAD 'input' AS (B: bag {T: tuple(v:INT)}); - -dump data; - -data2 = FOREACH data GENERATE NullToEmptyBag(B) as P; - -dump data2; - -STORE data2 INTO 'output'; - http://git-wip-us.apache.org/repos/asf/bigtop/blob/4cee56bd/bigtop-tests/test-artifacts/datafu/src/main/resources/datafu/bags/prependToBagTest.pig ---------------------------------------------------------------------- diff --git a/bigtop-tests/test-artifacts/datafu/src/main/resources/datafu/bags/prependToBagTest.pig b/bigtop-tests/test-artifacts/datafu/src/main/resources/datafu/bags/prependToBagTest.pig deleted file mode 100644 index c852346..0000000 --- a/bigtop-tests/test-artifacts/datafu/src/main/resources/datafu/bags/prependToBagTest.pig +++ /dev/null @@ -1,9 +0,0 @@ -register $JAR_PATH - -define PrependToBag datafu.pig.bags.PrependToBag(); - -data = LOAD 'input' AS (key:INT, B: bag{T: tuple(v:INT)}, T: tuple(v:INT)); - -data2 = FOREACH data GENERATE key, PrependToBag(B,T) as B; - -STORE data2 INTO 'output'; http://git-wip-us.apache.org/repos/asf/bigtop/blob/4cee56bd/bigtop-tests/test-artifacts/datafu/src/main/resources/datafu/bags/sets/setIntersectTest.pig ---------------------------------------------------------------------- diff --git a/bigtop-tests/test-artifacts/datafu/src/main/resources/datafu/bags/sets/setIntersectTest.pig b/bigtop-tests/test-artifacts/datafu/src/main/resources/datafu/bags/sets/setIntersectTest.pig deleted file mode 100644 index 6f590e8..0000000 --- a/bigtop-tests/test-artifacts/datafu/src/main/resources/datafu/bags/sets/setIntersectTest.pig +++ /dev/null @@ -1,9 +0,0 @@ -register $JAR_PATH - -define SetIntersect datafu.pig.bags.sets.SetIntersect(); - -data = LOAD 'input' AS (B1:bag{T:tuple(val1:int,val2:int)},B2:bag{T:tuple(val1:int,val2:int)}); - -data2 = FOREACH data GENERATE SetIntersect(B1,B2); - -STORE data2 INTO 'output'; http://git-wip-us.apache.org/repos/asf/bigtop/blob/4cee56bd/bigtop-tests/test-artifacts/datafu/src/main/resources/datafu/bags/sets/setUnionTest.pig ---------------------------------------------------------------------- diff --git a/bigtop-tests/test-artifacts/datafu/src/main/resources/datafu/bags/sets/setUnionTest.pig b/bigtop-tests/test-artifacts/datafu/src/main/resources/datafu/bags/sets/setUnionTest.pig deleted file mode 100644 index a5e1c4d..0000000 --- a/bigtop-tests/test-artifacts/datafu/src/main/resources/datafu/bags/sets/setUnionTest.pig +++ /dev/null @@ -1,13 +0,0 @@ -register $JAR_PATH - -define SetUnion datafu.pig.bags.sets.SetUnion(); - -data = LOAD 'input' AS (B1:bag{T:tuple(val1:int,val2:int)},B2:bag{T:tuple(val1:int,val2:int)}); - -dump data - -data2 = FOREACH data GENERATE SetUnion(B1,B2); - -dump data2 - -STORE data2 INTO 'output'; http://git-wip-us.apache.org/repos/asf/bigtop/blob/4cee56bd/bigtop-tests/test-artifacts/datafu/src/main/resources/datafu/bags/unorderedPairsTests.pig ---------------------------------------------------------------------- diff --git a/bigtop-tests/test-artifacts/datafu/src/main/resources/datafu/bags/unorderedPairsTests.pig b/bigtop-tests/test-artifacts/datafu/src/main/resources/datafu/bags/unorderedPairsTests.pig deleted file mode 100644 index 1bf68bd..0000000 --- a/bigtop-tests/test-artifacts/datafu/src/main/resources/datafu/bags/unorderedPairsTests.pig +++ /dev/null @@ -1,16 +0,0 @@ -register $JAR_PATH - -define UnorderedPairs datafu.pig.bags.UnorderedPairs(); - -data = LOAD 'input' AS (B: bag {T: tuple(v:INT)}); - -data2 = FOREACH data GENERATE UnorderedPairs(B) as P; - -data3 = FOREACH data2 GENERATE FLATTEN(P); - -data4 = FOREACH data3 GENERATE FLATTEN(elem1), FLATTEN(elem2); - -data5 = ORDER data4 BY $0, $1; - -STORE data5 INTO 'output'; - http://git-wip-us.apache.org/repos/asf/bigtop/blob/4cee56bd/bigtop-tests/test-artifacts/datafu/src/main/resources/datafu/bags/unorderedPairsTests2.pig ---------------------------------------------------------------------- diff --git a/bigtop-tests/test-artifacts/datafu/src/main/resources/datafu/bags/unorderedPairsTests2.pig b/bigtop-tests/test-artifacts/datafu/src/main/resources/datafu/bags/unorderedPairsTests2.pig deleted file mode 100644 index aada011..0000000 --- a/bigtop-tests/test-artifacts/datafu/src/main/resources/datafu/bags/unorderedPairsTests2.pig +++ /dev/null @@ -1,12 +0,0 @@ -register $JAR_PATH - -define UnorderedPairs datafu.pig.bags.UnorderedPairs(); - -data = LOAD 'input' AS (A:int, B: bag {T: tuple(v:INT)}); - -data2 = FOREACH data GENERATE A, UnorderedPairs(B) as P; - -data3 = FOREACH data2 GENERATE A, FLATTEN(P); - -STORE data3 INTO 'output'; - http://git-wip-us.apache.org/repos/asf/bigtop/blob/4cee56bd/bigtop-tests/test-artifacts/datafu/src/main/resources/datafu/date/timeCountPageViewsTest.pig ---------------------------------------------------------------------- diff --git a/bigtop-tests/test-artifacts/datafu/src/main/resources/datafu/date/timeCountPageViewsTest.pig b/bigtop-tests/test-artifacts/datafu/src/main/resources/datafu/date/timeCountPageViewsTest.pig deleted file mode 100644 index 1e23a41..0000000 --- a/bigtop-tests/test-artifacts/datafu/src/main/resources/datafu/date/timeCountPageViewsTest.pig +++ /dev/null @@ -1,13 +0,0 @@ -register $JAR_PATH - -define TimeCount datafu.pig.date.TimeCount('$TIME_WINDOW'); - -views = LOAD 'input' AS (user_id:int, page_id:int, time:chararray); - -views_grouped = GROUP views BY (user_id, page_id); -view_counts = foreach views_grouped { - views = order views by time; - generate group.user_id as user_id, group.page_id as page_id, TimeCount(views.(time)) as count; -} - -STORE view_counts INTO 'output'; http://git-wip-us.apache.org/repos/asf/bigtop/blob/4cee56bd/bigtop-tests/test-artifacts/datafu/src/main/resources/datafu/geo/haversineTest.pig ---------------------------------------------------------------------- diff --git a/bigtop-tests/test-artifacts/datafu/src/main/resources/datafu/geo/haversineTest.pig b/bigtop-tests/test-artifacts/datafu/src/main/resources/datafu/geo/haversineTest.pig deleted file mode 100644 index e52cc1f..0000000 --- a/bigtop-tests/test-artifacts/datafu/src/main/resources/datafu/geo/haversineTest.pig +++ /dev/null @@ -1,9 +0,0 @@ -register $JAR_PATH - -define HaversineDistInMiles datafu.pig.geo.HaversineDistInMiles(); - -data = LOAD 'input' AS (lat1:double,lng1:double,lat2:double,lng2:double); - -data2 = FOREACH data GENERATE HaversineDistInMiles(lat1,lng1,lat2,lng2); - -STORE data2 INTO 'output'; http://git-wip-us.apache.org/repos/asf/bigtop/blob/4cee56bd/bigtop-tests/test-artifacts/datafu/src/main/resources/datafu/hash/md5Base64Test.pig ---------------------------------------------------------------------- diff --git a/bigtop-tests/test-artifacts/datafu/src/main/resources/datafu/hash/md5Base64Test.pig b/bigtop-tests/test-artifacts/datafu/src/main/resources/datafu/hash/md5Base64Test.pig deleted file mode 100644 index 5a12c2e..0000000 --- a/bigtop-tests/test-artifacts/datafu/src/main/resources/datafu/hash/md5Base64Test.pig +++ /dev/null @@ -1,9 +0,0 @@ -register $JAR_PATH - -define MD5 datafu.pig.hash.MD5Base64(); - -data_in = LOAD 'input' as (val:chararray); - -data_out = FOREACH data_in GENERATE MD5(val) as val; - -STORE data_out INTO 'output'; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/bigtop/blob/4cee56bd/bigtop-tests/test-artifacts/datafu/src/main/resources/datafu/hash/md5Test.pig ---------------------------------------------------------------------- diff --git a/bigtop-tests/test-artifacts/datafu/src/main/resources/datafu/hash/md5Test.pig b/bigtop-tests/test-artifacts/datafu/src/main/resources/datafu/hash/md5Test.pig deleted file mode 100644 index 3fc6aaa..0000000 --- a/bigtop-tests/test-artifacts/datafu/src/main/resources/datafu/hash/md5Test.pig +++ /dev/null @@ -1,9 +0,0 @@ -register $JAR_PATH - -define MD5 datafu.pig.hash.MD5(); - -data_in = LOAD 'input' as (val:chararray); - -data_out = FOREACH data_in GENERATE MD5(val) as val; - -STORE data_out INTO 'output'; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/bigtop/blob/4cee56bd/bigtop-tests/test-artifacts/datafu/src/main/resources/datafu/linkanalysis/pageRankTest.pig ---------------------------------------------------------------------- diff --git a/bigtop-tests/test-artifacts/datafu/src/main/resources/datafu/linkanalysis/pageRankTest.pig b/bigtop-tests/test-artifacts/datafu/src/main/resources/datafu/linkanalysis/pageRankTest.pig deleted file mode 100644 index a0e439c..0000000 --- a/bigtop-tests/test-artifacts/datafu/src/main/resources/datafu/linkanalysis/pageRankTest.pig +++ /dev/null @@ -1,25 +0,0 @@ -register $JAR_PATH - -/* Need to enable dangling node handling since the Wikipedia example has them, - otherwise the ranks won't be right. */ -define PageRank datafu.pig.linkanalysis.PageRank('dangling_nodes','true'); - -data = LOAD 'input' AS (topic:INT,source:INT,dest:INT,weight:DOUBLE); - -data_grouped = GROUP data by (topic,source); - -data_grouped = foreach data_grouped { - generate group.topic as topic, group.source as source, data.(dest,weight) as edges; -}; - -data_grouped2 = GROUP data_grouped by topic; -data_grouped2 = foreach data_grouped2 { - generate group as topic, FLATTEN(PageRank(data_grouped.(source,edges))) as (source,rank); -}; - -data_grouped3 = FOREACH data_grouped2 GENERATE - topic, - source, - rank; - -STORE data_grouped3 INTO 'output'; http://git-wip-us.apache.org/repos/asf/bigtop/blob/4cee56bd/bigtop-tests/test-artifacts/datafu/src/main/resources/datafu/numbers/randomIntRangeTest.pig ---------------------------------------------------------------------- diff --git a/bigtop-tests/test-artifacts/datafu/src/main/resources/datafu/numbers/randomIntRangeTest.pig b/bigtop-tests/test-artifacts/datafu/src/main/resources/datafu/numbers/randomIntRangeTest.pig deleted file mode 100644 index 3ca45c7..0000000 --- a/bigtop-tests/test-artifacts/datafu/src/main/resources/datafu/numbers/randomIntRangeTest.pig +++ /dev/null @@ -1,8 +0,0 @@ -register $JAR_PATH - -define RandInt datafu.pig.numbers.RandInt(); - -data = LOAD 'input' AS (key:INT); -data2 = FOREACH data GENERATE key, RandInt($MIN,$MAX) as val; - -STORE data2 INTO 'output'; http://git-wip-us.apache.org/repos/asf/bigtop/blob/4cee56bd/bigtop-tests/test-artifacts/datafu/src/main/resources/datafu/sessions/sessionizeTest.pig ---------------------------------------------------------------------- diff --git a/bigtop-tests/test-artifacts/datafu/src/main/resources/datafu/sessions/sessionizeTest.pig b/bigtop-tests/test-artifacts/datafu/src/main/resources/datafu/sessions/sessionizeTest.pig deleted file mode 100644 index 6a4939e..0000000 --- a/bigtop-tests/test-artifacts/datafu/src/main/resources/datafu/sessions/sessionizeTest.pig +++ /dev/null @@ -1,17 +0,0 @@ -register $JAR_PATH - -define Sessionize datafu.pig.sessions.Sessionize('$TIME_WINDOW'); - -views = LOAD 'input' AS (time:chararray, user_id:int, value:int); - -views_grouped = GROUP views BY user_id; -view_counts = FOREACH views_grouped { - views = ORDER views BY time; - GENERATE flatten(Sessionize(views)) as (time,user_id,value,session_id); -} - -max_value = GROUP view_counts BY (user_id, session_id); - -max_value = FOREACH max_value GENERATE group.user_id, MAX(view_counts.value) AS val; - -STORE max_value INTO 'output'; http://git-wip-us.apache.org/repos/asf/bigtop/blob/4cee56bd/bigtop-tests/test-artifacts/datafu/src/main/resources/datafu/stats/markovPairDefault.pig ---------------------------------------------------------------------- diff --git a/bigtop-tests/test-artifacts/datafu/src/main/resources/datafu/stats/markovPairDefault.pig b/bigtop-tests/test-artifacts/datafu/src/main/resources/datafu/stats/markovPairDefault.pig deleted file mode 100644 index a121cb1..0000000 --- a/bigtop-tests/test-artifacts/datafu/src/main/resources/datafu/stats/markovPairDefault.pig +++ /dev/null @@ -1,14 +0,0 @@ -register $JAR_PATH - -define markovPairs datafu.pig.stats.MarkovPairs(); - -data = load 'input' as $schema; -describe data; - -data_out1 = foreach data generate data as orig_bag; -describe data_out1; - -data_out = foreach data_out1 generate markovPairs(orig_bag) as markov_bag; -describe data_out; - -store data_out into 'output'; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/bigtop/blob/4cee56bd/bigtop-tests/test-artifacts/datafu/src/main/resources/datafu/stats/markovPairLookahead.pig ---------------------------------------------------------------------- diff --git a/bigtop-tests/test-artifacts/datafu/src/main/resources/datafu/stats/markovPairLookahead.pig b/bigtop-tests/test-artifacts/datafu/src/main/resources/datafu/stats/markovPairLookahead.pig deleted file mode 100644 index 269a1bc..0000000 --- a/bigtop-tests/test-artifacts/datafu/src/main/resources/datafu/stats/markovPairLookahead.pig +++ /dev/null @@ -1,14 +0,0 @@ -register $JAR_PATH - -define markovPairs datafu.pig.stats.MarkovPairs('$lookahead'); - -data = load 'input' as $schema; -describe data; - -data_out1 = foreach data generate data as orig_bag; -describe data_out1; - -data_out = foreach data_out1 generate markovPairs(orig_bag) as markov_bag; -describe data_out; - -store data_out into 'output'; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/bigtop/blob/4cee56bd/bigtop-tests/test-artifacts/datafu/src/main/resources/datafu/stats/medianTest.pig ---------------------------------------------------------------------- diff --git a/bigtop-tests/test-artifacts/datafu/src/main/resources/datafu/stats/medianTest.pig b/bigtop-tests/test-artifacts/datafu/src/main/resources/datafu/stats/medianTest.pig deleted file mode 100644 index 0a439ce..0000000 --- a/bigtop-tests/test-artifacts/datafu/src/main/resources/datafu/stats/medianTest.pig +++ /dev/null @@ -1,21 +0,0 @@ -register $JAR_PATH - -define Median datafu.pig.stats.Median(); - -data_in = LOAD 'input' as (val:int); - -/*describe data_in;*/ - -data_out = GROUP data_in ALL; - -/*describe data_out;*/ - -data_out = FOREACH data_out { - sorted = ORDER data_in BY val; - GENERATE Median(sorted) as medians; -} -data_out = FOREACH data_out GENERATE FLATTEN(medians); - -/*describe data_out;*/ - -STORE data_out into 'output'; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/bigtop/blob/4cee56bd/bigtop-tests/test-artifacts/datafu/src/main/resources/datafu/stats/quantileTest.pig ---------------------------------------------------------------------- diff --git a/bigtop-tests/test-artifacts/datafu/src/main/resources/datafu/stats/quantileTest.pig b/bigtop-tests/test-artifacts/datafu/src/main/resources/datafu/stats/quantileTest.pig deleted file mode 100644 index 604d179..0000000 --- a/bigtop-tests/test-artifacts/datafu/src/main/resources/datafu/stats/quantileTest.pig +++ /dev/null @@ -1,21 +0,0 @@ -register $JAR_PATH - -define Quantile datafu.pig.stats.Quantile($QUANTILES); - -data_in = LOAD 'input' as (val:int); - -/*describe data_in;*/ - -data_out = GROUP data_in ALL; - -/*describe data_out;*/ - -data_out = FOREACH data_out { - sorted = ORDER data_in BY val; - GENERATE Quantile(sorted) as quantiles; -} -data_out = FOREACH data_out GENERATE FLATTEN(quantiles); - -/*describe data_out;*/ - -STORE data_out into 'output'; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/bigtop/blob/4cee56bd/bigtop-tests/test-artifacts/datafu/src/main/resources/datafu/stats/streamingMedianTest.pig ---------------------------------------------------------------------- diff --git a/bigtop-tests/test-artifacts/datafu/src/main/resources/datafu/stats/streamingMedianTest.pig b/bigtop-tests/test-artifacts/datafu/src/main/resources/datafu/stats/streamingMedianTest.pig deleted file mode 100644 index 27d64f3..0000000 --- a/bigtop-tests/test-artifacts/datafu/src/main/resources/datafu/stats/streamingMedianTest.pig +++ /dev/null @@ -1,21 +0,0 @@ -register $JAR_PATH - -define Median datafu.pig.stats.StreamingMedian(); - -data_in = LOAD 'input' as (val:int); - -/*describe data_in;*/ - -data_out = GROUP data_in ALL; - -/*describe data_out;*/ - -data_out = FOREACH data_out { - sorted = ORDER data_in BY val; - GENERATE Median(sorted) as medians; -} -data_out = FOREACH data_out GENERATE FLATTEN(medians); - -/*describe data_out;*/ - -STORE data_out into 'output'; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/bigtop/blob/4cee56bd/bigtop-tests/test-artifacts/datafu/src/main/resources/datafu/stats/streamingQuantileTest.pig ---------------------------------------------------------------------- diff --git a/bigtop-tests/test-artifacts/datafu/src/main/resources/datafu/stats/streamingQuantileTest.pig b/bigtop-tests/test-artifacts/datafu/src/main/resources/datafu/stats/streamingQuantileTest.pig deleted file mode 100644 index 51c3bc5..0000000 --- a/bigtop-tests/test-artifacts/datafu/src/main/resources/datafu/stats/streamingQuantileTest.pig +++ /dev/null @@ -1,18 +0,0 @@ -register $JAR_PATH - -define Quantile datafu.pig.stats.StreamingQuantile($QUANTILES); - -data_in = LOAD 'input' as (val:int); - -/*describe data_in;*/ - -data_out = GROUP data_in ALL; - -/*describe data_out;*/ - -data_out = FOREACH data_out GENERATE Quantile(data_in.val) as quantiles; -data_out = FOREACH data_out GENERATE FLATTEN(quantiles); - -/*describe data_out;*/ - -STORE data_out into 'output'; http://git-wip-us.apache.org/repos/asf/bigtop/blob/4cee56bd/bigtop-tests/test-artifacts/datafu/src/main/resources/datafu/stats/wilsonBinConfTests.pig ---------------------------------------------------------------------- diff --git a/bigtop-tests/test-artifacts/datafu/src/main/resources/datafu/stats/wilsonBinConfTests.pig b/bigtop-tests/test-artifacts/datafu/src/main/resources/datafu/stats/wilsonBinConfTests.pig deleted file mode 100644 index 19fa466..0000000 --- a/bigtop-tests/test-artifacts/datafu/src/main/resources/datafu/stats/wilsonBinConfTests.pig +++ /dev/null @@ -1,11 +0,0 @@ -register $JAR_PATH - -define WilsonBinConf datafu.pig.stats.WilsonBinConf('$alpha'); - -data = load 'input' as (successes:long, totals:long); -describe data; - -data_out = FOREACH data GENERATE WilsonBinConf(successes, totals) as interval; -data_out = FOREACH data_out GENERATE FLATTEN(interval); - -store data_out into 'output'; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/bigtop/blob/4cee56bd/bigtop-tests/test-artifacts/datafu/src/main/resources/datafu/urls/userAgentTest.pig ---------------------------------------------------------------------- diff --git a/bigtop-tests/test-artifacts/datafu/src/main/resources/datafu/urls/userAgentTest.pig b/bigtop-tests/test-artifacts/datafu/src/main/resources/datafu/urls/userAgentTest.pig deleted file mode 100644 index 4548755..0000000 --- a/bigtop-tests/test-artifacts/datafu/src/main/resources/datafu/urls/userAgentTest.pig +++ /dev/null @@ -1,8 +0,0 @@ -register $JAR_PATH - -define UserAgentClassify datafu.pig.urls.UserAgentClassify(); - -data = load 'input' as (usr_agent:chararray); -data_out = foreach data generate UserAgentClassify(usr_agent) as class; -describe data_out; -store data_out into 'output'; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/bigtop/blob/4cee56bd/bigtop-tests/test-artifacts/datafu/src/main/resources/datafu/util/assertWithMessageTest.pig ---------------------------------------------------------------------- diff --git a/bigtop-tests/test-artifacts/datafu/src/main/resources/datafu/util/assertWithMessageTest.pig b/bigtop-tests/test-artifacts/datafu/src/main/resources/datafu/util/assertWithMessageTest.pig deleted file mode 100644 index f240987..0000000 --- a/bigtop-tests/test-artifacts/datafu/src/main/resources/datafu/util/assertWithMessageTest.pig +++ /dev/null @@ -1,10 +0,0 @@ -register $JAR_PATH - -define ASSERT datafu.pig.util.ASSERT(); - -data = LOAD 'input' AS (val:INT); - -data2 = FILTER data BY ASSERT(val,'assertion appears to have failed, doh!'); - -STORE data2 INTO 'output'; - http://git-wip-us.apache.org/repos/asf/bigtop/blob/4cee56bd/bigtop-tests/test-artifacts/datafu/src/main/resources/datafu/util/assertWithoutMessageTest.pig ---------------------------------------------------------------------- diff --git a/bigtop-tests/test-artifacts/datafu/src/main/resources/datafu/util/assertWithoutMessageTest.pig b/bigtop-tests/test-artifacts/datafu/src/main/resources/datafu/util/assertWithoutMessageTest.pig deleted file mode 100644 index c6368e7..0000000 --- a/bigtop-tests/test-artifacts/datafu/src/main/resources/datafu/util/assertWithoutMessageTest.pig +++ /dev/null @@ -1,10 +0,0 @@ -register $JAR_PATH - -define ASSERT datafu.pig.util.ASSERT(); - -data = LOAD 'input' AS (val:INT); - -data2 = FILTER data BY ASSERT(val); - -STORE data2 INTO 'output'; - http://git-wip-us.apache.org/repos/asf/bigtop/blob/4cee56bd/bigtop-tests/test-artifacts/datafu/src/main/resources/datafu/util/intToBoolTest.pig ---------------------------------------------------------------------- diff --git a/bigtop-tests/test-artifacts/datafu/src/main/resources/datafu/util/intToBoolTest.pig b/bigtop-tests/test-artifacts/datafu/src/main/resources/datafu/util/intToBoolTest.pig deleted file mode 100644 index 18cda42..0000000 --- a/bigtop-tests/test-artifacts/datafu/src/main/resources/datafu/util/intToBoolTest.pig +++ /dev/null @@ -1,10 +0,0 @@ -register $JAR_PATH - -define IntToBool datafu.pig.util.IntToBool(); - -data = LOAD 'input' AS (val:INT); - -data2 = FOREACH data GENERATE IntToBool(val); - -STORE data2 INTO 'output'; - http://git-wip-us.apache.org/repos/asf/bigtop/blob/4cee56bd/bigtop-tests/test-artifacts/datafu/src/main/resources/datafu/util/intToBoolToIntTest.pig ---------------------------------------------------------------------- diff --git a/bigtop-tests/test-artifacts/datafu/src/main/resources/datafu/util/intToBoolToIntTest.pig b/bigtop-tests/test-artifacts/datafu/src/main/resources/datafu/util/intToBoolToIntTest.pig deleted file mode 100644 index 82d3ee0..0000000 --- a/bigtop-tests/test-artifacts/datafu/src/main/resources/datafu/util/intToBoolToIntTest.pig +++ /dev/null @@ -1,12 +0,0 @@ -register $JAR_PATH - -define IntToBool datafu.pig.util.IntToBool(); -define BoolToInt datafu.pig.util.BoolToInt(); - -data = LOAD 'input' AS (val:INT); - -data2 = FOREACH data GENERATE IntToBool(val) as val; -data3 = FOREACH data2 GENERATE BoolToInt(val) as val; - -STORE data3 INTO 'output'; - http://git-wip-us.apache.org/repos/asf/bigtop/blob/4cee56bd/bigtop-tests/test-artifacts/package/src/main/resources/package_data.xml ---------------------------------------------------------------------- diff --git a/bigtop-tests/test-artifacts/package/src/main/resources/package_data.xml b/bigtop-tests/test-artifacts/package/src/main/resources/package_data.xml index 19365e8..d9afcb3 100644 --- a/bigtop-tests/test-artifacts/package/src/main/resources/package_data.xml +++ b/bigtop-tests/test-artifacts/package/src/main/resources/package_data.xml @@ -446,25 +446,6 @@ easy to test, and efficient to run.</description> <hive-hcatalog>/self</hive-hcatalog> </deps> </hive-hcatalog-server> - <pig-udf-datafu> - <metadata> - <summary>A collection of user-defined functions for Hadoop and Pig.</summary> - <description> DataFu is a collection of user-defined functions for working with large-scale - data in Hadoop and Pig. This library was born out of the need for a stable, - well-tested library of UDFs for data mining and statistics. It is used - at LinkedIn in many of our off-line workflows for data derived products like - "People You May Know" and "Skills". - - It contains functions for: PageRank, Quantiles (median), variance, Sessionization, - Convenience bag functions (e.g., set operations, enumerating bags, etc), - Convenience utility functions (e.g., assertions, easier writing of EvalFuncs) - and more...</description> - <url>https://github.com/linkedin/datafu</url> - </metadata> - <deps> - <pig/> - </deps> - </pig-udf-datafu> <hive-jdbc> <metadata> <summary>Provides libraries necessary to connect to Apache Hive via JDBC</summary> http://git-wip-us.apache.org/repos/asf/bigtop/blob/4cee56bd/bigtop-tests/test-artifacts/pom.xml ---------------------------------------------------------------------- diff --git a/bigtop-tests/test-artifacts/pom.xml b/bigtop-tests/test-artifacts/pom.xml index f8dea78..b0af51e 100644 --- a/bigtop-tests/test-artifacts/pom.xml +++ b/bigtop-tests/test-artifacts/pom.xml @@ -45,7 +45,6 @@ <module>hue</module> <module>solr</module> <module>crunch</module> - <module>datafu</module> <module>longevity</module> <module>hcatalog</module> <module>spark</module> http://git-wip-us.apache.org/repos/asf/bigtop/blob/4cee56bd/bigtop-tests/test-execution/smokes/datafu/pom.xml ---------------------------------------------------------------------- diff --git a/bigtop-tests/test-execution/smokes/datafu/pom.xml b/bigtop-tests/test-execution/smokes/datafu/pom.xml deleted file mode 100644 index 45b007f..0000000 --- a/bigtop-tests/test-execution/smokes/datafu/pom.xml +++ /dev/null @@ -1,140 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<!-- - Licensed to the Apache Software Foundation (ASF) under one or more - contributor license agreements. See the NOTICE file distributed with - this work for additional information regarding copyright ownership. - The ASF licenses this file to You under the Apache License, Version 2.0 - (the "License"); you may not use this file except in compliance with - the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. ---> -<project xmlns="http://maven.apache.org/POM/4.0.0"; - xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; - xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd";> - <modelVersion>4.0.0</modelVersion> - - <parent> - <groupId>org.apache.bigtop.itest</groupId> - <artifactId>smoke-tests</artifactId> - <version>1.3.1-SNAPSHOT</version> - <relativePath>../pom.xml</relativePath> - </parent> - - <groupId>org.apache.bigtop.itest</groupId> - <artifactId>datafu-smoke-execution</artifactId> - <version>1.3.1-SNAPSHOT</version> - <name>datafu smoke test execution</name> - - <properties> - <org.apache.maven-dependency-plugin.groupId>org.apache.bigtop.itest</org.apache.maven-dependency-plugin.groupId> - <org.apache.maven-dependency-plugin.artifactId>datafu-smoke</org.apache.maven-dependency-plugin.artifactId> - <org.apache.maven-dependency-plugin.version>${project.version}</org.apache.maven-dependency-plugin.version> - <org.apache.maven-dependency-plugin.output>${project.build.directory}</org.apache.maven-dependency-plugin.output> - <org.apache.maven-dependency-plugin.type>jar</org.apache.maven-dependency-plugin.type> - <org.apache.maven-failsafe-plugin.testInclude>**/*Tests*</org.apache.maven-failsafe-plugin.testInclude> - - <HADOOP_MAPRED_HOME>${env.HADOOP_MAPRED_HOME}</HADOOP_MAPRED_HOME> - <HADOOP_CONF_DIR>${env.HADOOP_CONF_DIR}</HADOOP_CONF_DIR> - <PIG_HOME>${env.PIG_HOME}</PIG_HOME> - </properties> - - <dependencies> - <dependency> - <groupId>${org.apache.maven-dependency-plugin.groupId}</groupId> - <artifactId>${org.apache.maven-dependency-plugin.artifactId}</artifactId> - <version>${project.version}</version> - </dependency> - </dependencies> - - <build> - <plugins> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-install-plugin</artifactId> - <executions> - <execution> - <phase>initialize</phase> - <goals> - <goal>install-file</goal> - </goals> - </execution> - </executions> - <configuration> - <file>${PIG_HOME}/pig.jar</file> - <groupId>org.apache.pig</groupId> - <artifactId>pig</artifactId> - <version>${pig.version}</version> - <packaging>jar</packaging> - </configuration> - </plugin> - - - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-dependency-plugin</artifactId> - </plugin> - - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-enforcer-plugin</artifactId> - <version>1.0</version> - <executions> - <execution> - <id>enforce-property</id> - <goals> - <goal>enforce</goal> - </goals> - <configuration> - <rules> - <requireProperty> - <property>HADOOP_MAPRED_HOME</property> - <message>HADOOP_MAPRED_HOME env. variable has to be set</message> - </requireProperty> - <requireProperty> - <property>HADOOP_CONF_DIR</property> - <message>HADOOP_CONF_DIR env. variable has to be set</message> - </requireProperty> - <requireProperty> - <property>PIG_HOME</property> - <message>PIG_HOME env. variable has to be set</message> - </requireProperty> - </rules> - <fail>true</fail> - </configuration> - </execution> - </executions> - </plugin> - - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-failsafe-plugin</artifactId> - <version>2.11</version> - <configuration> - <forkMode>always</forkMode> - <argLine>-Dpigunit.exectype.cluster=true</argLine> - <additionalClasspathElements> - <additionalClasspathElement>${HADOOP_CONF_DIR}</additionalClasspathElement> - </additionalClasspathElements> - <systemPropertyVariables> - <datafu.jar.dir>${PIG_HOME}</datafu.jar.dir> - </systemPropertyVariables> - </configuration> - - <!-- Disabling for now: configuration> - <testSourceDirectory>src</testSourceDirectory> - <testClassesDirectory>target/classes</testClassesDirectory> - <skipTests>false</skipTests> - <testFailureIgnore>false</testFailureIgnore> - <argLine>-Dsun.lang.ClassLoader.allowArraySyntax=true -Djava.endorsed.dirs=${project.build.testOutputDirectory}/endorsed</argLine> - </configuration --> - </plugin> - </plugins> - </build> -</project> http://git-wip-us.apache.org/repos/asf/bigtop/blob/4cee56bd/bigtop.bom ---------------------------------------------------------------------- diff --git a/bigtop.bom b/bigtop.bom index 02e5b96..3664f7f 100644 --- a/bigtop.bom +++ b/bigtop.bom @@ -252,17 +252,6 @@ bigtop { site = "${apache.APACHE_MIRROR}/${download_path}" archive = "${apache.APACHE_ARCHIVE}/${download_path}" } } - 'datafu' { - name = 'datafu' - pkg = 'pig-udf-datafu' - relNotes = 'Pig UDF Datafu' - version { base = '1.3.0'; pkg = base; release = 1 } - tarball { destination = "$name-${version.base}.tar.gz" - source = "apache-$name-incubating-sources-${version.base}.tgz" } - url { download_path = "incubator/$name/apache-$name-incubating-${version.base}" - site = "${apache.APACHE_MIRROR}/${download_path}" - archive = "${apache.APACHE_ARCHIVE}/${download_path}" } - } 'solr' { name = 'solr' relNotes = 'Apache Solr'
