[CALCITE-1561] Make PigTest cluster aware of data files; hopefully this will prevent intermittent test failures (Eli Levine)
Also add dependencies. Close apache/calcite#397 Project: http://git-wip-us.apache.org/repos/asf/calcite/repo Commit: http://git-wip-us.apache.org/repos/asf/calcite/commit/713c8f4f Tree: http://git-wip-us.apache.org/repos/asf/calcite/tree/713c8f4f Diff: http://git-wip-us.apache.org/repos/asf/calcite/diff/713c8f4f Branch: refs/heads/master Commit: 713c8f4fc9931b7118f393b9348441f25e36c9a2 Parents: e0a1f7d Author: Eli Levine <[email protected]> Authored: Thu Mar 9 16:03:27 2017 -0800 Committer: Julian Hyde <[email protected]> Committed: Tue Mar 14 21:42:14 2017 -0700 ---------------------------------------------------------------------- pig/pom.xml | 5 +++ .../calcite/test/PigRelBuilderStyleTest.java | 47 +++++++++++++------- 2 files changed, 36 insertions(+), 16 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/calcite/blob/713c8f4f/pig/pom.xml ---------------------------------------------------------------------- diff --git a/pig/pom.xml b/pig/pom.xml index 169cdc7..2ec910d 100644 --- a/pig/pom.xml +++ b/pig/pom.xml @@ -57,6 +57,11 @@ limitations under the License. <scope>test</scope> </dependency> <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-common</artifactId> + <scope>test</scope> + </dependency> + <dependency> <groupId>com.google.guava</groupId> <artifactId>guava</artifactId> </dependency> http://git-wip-us.apache.org/repos/asf/calcite/blob/713c8f4f/pig/src/test/java/org/apache/calcite/test/PigRelBuilderStyleTest.java ---------------------------------------------------------------------- diff --git a/pig/src/test/java/org/apache/calcite/test/PigRelBuilderStyleTest.java b/pig/src/test/java/org/apache/calcite/test/PigRelBuilderStyleTest.java index b974b13..08d8363 100644 --- a/pig/src/test/java/org/apache/calcite/test/PigRelBuilderStyleTest.java +++ b/pig/src/test/java/org/apache/calcite/test/PigRelBuilderStyleTest.java @@ -36,9 +36,13 @@ import org.apache.calcite.tools.Frameworks; import org.apache.calcite.tools.RelBuilder; import org.apache.calcite.tools.RelBuilderFactory; +import org.apache.hadoop.fs.Path; +import org.apache.pig.pigunit.Cluster; import org.apache.pig.pigunit.PigTest; +import org.apache.pig.test.Util; import org.junit.After; +import org.junit.Before; import org.junit.Test; import static org.apache.calcite.rel.rules.FilterJoinRule.TRUE_PREDICATE; @@ -62,7 +66,7 @@ public class PigRelBuilderStyleTest extends AbstractPigTest { .filter(builder.call(GREATER_THAN, builder.field("tc0"), builder.literal("abc"))).build(); final RelNode optimized = optimizeWithVolcano(node); assertScriptAndResults("t", getPigScript(optimized, schema), - "t = LOAD '" + getFullPathForTestDataFile("data.txt") + "t = LOAD 'data.txt" + "' USING PigStorage() AS (tc0:chararray, tc1:chararray);\n" + "t = FILTER t BY (tc0 > 'abc');", new String[] { "(b,2)", "(c,3)" }); @@ -79,7 +83,7 @@ public class PigRelBuilderStyleTest extends AbstractPigTest { .build(); final RelNode optimized = optimizeWithVolcano(node); assertScriptAndResults("t", getPigScript(optimized, schema), - "t = LOAD '" + getFullPathForTestDataFile("data.txt") + "t = LOAD 'data.txt" + "' USING PigStorage() AS (tc0:chararray, tc1:chararray);\n" + "t = FILTER t BY (tc0 > 'abc') AND (tc1 == '3');", new String[] { "(c,3)" }); @@ -94,7 +98,7 @@ public class PigRelBuilderStyleTest extends AbstractPigTest { .build(); final RelNode optimized = optimizeWithVolcano(node); assertScriptAndResults("t", getPigScript(optimized, schema), - "t = LOAD '" + getFullPathForTestDataFile("data.txt") + "t = LOAD 'data.txt" + "' USING PigStorage() AS (tc0:chararray, tc1:chararray);\n" + "t = GROUP t BY (tc0);\n" + "t = FOREACH t {\n" @@ -111,7 +115,7 @@ public class PigRelBuilderStyleTest extends AbstractPigTest { .aggregate(builder.groupKey(), builder.count(false, "c", builder.field("tc0"))).build(); final RelNode optimized = optimizeWithVolcano(node); assertScriptAndResults("t", getPigScript(optimized, schema), - "t = LOAD '" + getFullPathForTestDataFile("data.txt") + "t = LOAD 'data.txt" + "' USING PigStorage() AS (tc0:chararray, tc1:chararray);\n" + "t = GROUP t ALL;\n" + "t = FOREACH t {\n" @@ -129,7 +133,7 @@ public class PigRelBuilderStyleTest extends AbstractPigTest { .build(); final RelNode optimized = optimizeWithVolcano(node); assertScriptAndResults("t", getPigScript(optimized, schema), - "t = LOAD '" + getFullPathForTestDataFile("data.txt") + "t = LOAD 'data.txt" + "' USING PigStorage() AS (tc0:chararray, tc1:chararray);\n" + "t = GROUP t BY (tc0, tc1);\n" + "t = FOREACH t {\n" @@ -147,7 +151,7 @@ public class PigRelBuilderStyleTest extends AbstractPigTest { .build(); final RelNode optimized = optimizeWithVolcano(node); assertScriptAndResults("t", getPigScript(optimized, schema), - "t = LOAD '" + getFullPathForTestDataFile("data.txt") + "t = LOAD 'data.txt" + "' USING PigStorage() AS (tc0:chararray, tc1:chararray);\n" + "t = GROUP t BY (tc0, tc1);\n" + "t = FOREACH t {\n" @@ -167,11 +171,10 @@ public class PigRelBuilderStyleTest extends AbstractPigTest { .filter(builder.call(GREATER_THAN, builder.field("tc0"), builder.literal("a"))).build(); final RelNode optimized = optimizeWithVolcano(node); assertScriptAndResults("t", getPigScript(optimized, schema), - "t = LOAD '" + getFullPathForTestDataFile("data.txt") + "t = LOAD 'data.txt" + "' USING PigStorage() AS (tc0:chararray, tc1:chararray);\n" + "t = FILTER t BY (tc0 > 'a');\n" - + "s = LOAD '" - + getFullPathForTestDataFile("data2.txt") + + "s = LOAD 'data2.txt" + "' USING PigStorage() AS (sc0:chararray, sc1:chararray);\n" + "t = JOIN t BY tc1 , s BY sc0;", new String[] { "(b,2,2,label2)" }); @@ -189,11 +192,10 @@ public class PigRelBuilderStyleTest extends AbstractPigTest { .build(); final RelNode optimized = optimizeWithVolcano(node); assertScriptAndResults("t", getPigScript(optimized, schema), - "t = LOAD '" + getFullPathForTestDataFile("data.txt") + "t = LOAD 'data.txt" + "' USING PigStorage() AS (tc0:chararray, tc1:chararray);\n" + "t = FILTER t BY (tc0 > 'abc');\n" - + "s = LOAD '" - + getFullPathForTestDataFile("data2.txt") + + "s = LOAD 'data2.txt" + "' USING PigStorage() AS (sc0:chararray, sc1:chararray);\n" + "t = JOIN t BY tc1 LEFT, s BY sc0;\n" + "t = GROUP t BY (tc1);\n" @@ -206,10 +208,10 @@ public class PigRelBuilderStyleTest extends AbstractPigTest { private SchemaPlus createTestSchema() { SchemaPlus result = Frameworks.createRootSchema(false); result.add("t", - new PigTable(getFullPathForTestDataFile("data.txt"), + new PigTable("data.txt", new String[] { "tc0", "tc1" })); result.add("s", - new PigTable(getFullPathForTestDataFile("data2.txt"), + new PigTable("data2.txt", new String[] { "sc0", "sc1" })); return result; } @@ -242,8 +244,8 @@ public class PigRelBuilderStyleTest extends AbstractPigTest { return planner; } - private void assertScriptAndResults(String relAliasForStore, String script, String expectedScript, - String[] expectedResults) { + private void assertScriptAndResults(String relAliasForStore, String script, + String expectedScript, String[] expectedResults) { try { assertEquals(expectedScript, script); script = script + "\nSTORE " + relAliasForStore + " INTO 'myoutput';"; @@ -264,6 +266,19 @@ public class PigRelBuilderStyleTest extends AbstractPigTest { public void shutdownPigServer() { PigTest.getPigServer().shutdown(); } + + @Before + public void setupDataFilesForPigServer() throws Exception { + System.getProperties().setProperty("pigunit.exectype", + Util.getLocalTestMode().toString()); + Cluster cluster = PigTest.getCluster(); + cluster.update( + new Path(getFullPathForTestDataFile("data.txt")), + new Path("data.txt")); + cluster.update( + new Path(getFullPathForTestDataFile("data2.txt")), + new Path("data2.txt")); + } } // End PigRelBuilderStyleTest.java
