Modified: pig/branches/spark/test/org/apache/pig/test/TestMRJobStats.java URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/test/TestMRJobStats.java?rev=1784224&r1=1784223&r2=1784224&view=diff ============================================================================== --- pig/branches/spark/test/org/apache/pig/test/TestMRJobStats.java (original) +++ pig/branches/spark/test/org/apache/pig/test/TestMRJobStats.java Fri Feb 24 03:34:37 2017 @@ -102,7 +102,7 @@ public class TestMRJobStats { try { Constructor<MRJobStats> con = MRJobStats.class.getDeclaredConstructor(String.class, JobGraph.class); con.setAccessible(true); - MRJobStats jobStats = con.newInstance(name, plan); + MRJobStats jobStats = (MRJobStats) con.newInstance(name, plan); return jobStats; } catch (Exception e) { return null; @@ -202,49 +202,14 @@ public class TestMRJobStats { } } - private POStore createPOStoreForFileBasedSystemWithSubDirectories(long size, StoreFuncInterface storeFunc, Configuration conf) throws Exception { - File root = createTmpDirectory("outputRoot", null); - File dir1 = createTmpDirectory("dir1", root); - File dir2 = createTmpDirectory("dir2", root); - createTmpFile("tempFile1", size, dir1); - createTmpFile("tempFile2", size, dir2); - - storeFunc.setStoreLocation(root.getAbsolutePath(), new Job(conf)); - FuncSpec funcSpec = new FuncSpec(storeFunc.getClass().getCanonicalName()); - POStore poStore = new POStore(new OperatorKey()); - poStore.setSFile(new FileSpec(root.getAbsolutePath(), funcSpec)); - poStore.setStoreFunc(storeFunc); - poStore.setUp(); - - return poStore; - } - - private static File createTmpDirectory(String name, File root) throws Exception { - File directory = File.createTempFile(name, "", root); - - if (!(directory.delete())) { - throw new IOException("Could not delete temp file: " + directory.getAbsolutePath()); - } - - if (!(directory.mkdir())) { - throw new IOException("Could not create temp directory: " + directory.getAbsolutePath()); - } - - return directory; - } + private static POStore createPOStoreForFileBasedSystem(long size, StoreFuncInterface storeFunc, + Configuration conf) throws Exception { - private static File createTmpFile(String name, long size, File directory) throws Exception { - File file = directory == null ? File.createTempFile(name, ".tmp") : File.createTempFile(name, ".tmp", directory); + File file = File.createTempFile("tempFile", ".tmp"); file.deleteOnExit(); RandomAccessFile f = new RandomAccessFile(file, "rw"); f.setLength(size); f.close(); - return file; - } - - private static POStore createPOStoreForFileBasedSystem(long size, StoreFuncInterface storeFunc, - Configuration conf) throws Exception { - File file = createTmpFile("tempFile", size, null); storeFunc.setStoreLocation(file.getAbsolutePath(), new Job(conf)); FuncSpec funcSpec = new FuncSpec(storeFunc.getClass().getCanonicalName()); @@ -271,7 +236,7 @@ public class TestMRJobStats { } @Test - public void testGetOutputSizeUsingFileBasedStorage() throws Exception { + public void testGetOuputSizeUsingFileBasedStorage() throws Exception { // By default, FileBasedOutputSizeReader is used to compute the size of output. Configuration conf = new Configuration(); @@ -284,20 +249,7 @@ public class TestMRJobStats { } @Test - public void testGetOutputSizeUsingFileBasedStorageWithSubDirectories() throws Exception { - // By default, FileBasedOutputSizeReader is used to compute the size of output. - Configuration conf = new Configuration(); - - long size = 2L * 1024 * 1024 * 1024; - long outputSize = JobStats.getOutputSize( - createPOStoreForFileBasedSystemWithSubDirectories(size, new PigStorageWithStatistics(), conf), conf); - - assertEquals("The returned output size is expected to be sum of file sizes in the sub-directories", - 2 * size, outputSize); - } - - @Test - public void testGetOutputSizeUsingNonFileBasedStorage1() throws Exception { + public void testGetOuputSizeUsingNonFileBasedStorage1() throws Exception { // By default, FileBasedOutputSizeReader is used to compute the size of output. Configuration conf = new Configuration(); @@ -311,7 +263,7 @@ public class TestMRJobStats { } @Test - public void testGetOutputSizeUsingNonFileBasedStorage2() throws Exception { + public void testGetOuputSizeUsingNonFileBasedStorage2() throws Exception { // Register a custom output size reader in configuration Configuration conf = new Configuration(); conf.set(PigStatsOutputSizeReader.OUTPUT_SIZE_READER_KEY, @@ -327,7 +279,7 @@ public class TestMRJobStats { } @Test(expected = RuntimeException.class) - public void testGetOutputSizeUsingNonFileBasedStorage3() throws Exception { + public void testGetOuputSizeUsingNonFileBasedStorage3() throws Exception { // Register an invalid output size reader in configuration, and verify // that an exception is thrown at run-time. Configuration conf = new Configuration(); @@ -340,7 +292,7 @@ public class TestMRJobStats { } @Test - public void testGetOutputSizeUsingNonFileBasedStorage4() throws Exception { + public void testGetOuputSizeUsingNonFileBasedStorage4() throws Exception { // Register a comma-separated list of readers in configuration, and // verify that the one that supports a non-file-based uri is used. Configuration conf = new Configuration(); @@ -358,7 +310,7 @@ public class TestMRJobStats { } @Test - public void testGetOutputSizeUsingNonFileBasedStorage5() throws Exception { + public void testGetOuputSizeUsingNonFileBasedStorage5() throws Exception { Configuration conf = new Configuration(); long size = 2L * 1024 * 1024 * 1024;
Modified: pig/branches/spark/test/org/apache/pig/test/TestMacroExpansion.java URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/test/TestMacroExpansion.java?rev=1784224&r1=1784223&r2=1784224&view=diff ============================================================================== --- pig/branches/spark/test/org/apache/pig/test/TestMacroExpansion.java (original) +++ pig/branches/spark/test/org/apache/pig/test/TestMacroExpansion.java Fri Feb 24 03:34:37 2017 @@ -1187,33 +1187,6 @@ public class TestMacroExpansion { validateFailure(sb.toString(), expectedErr, "at"); } - - @Test - public void lineNumberTest3() throws Throwable { - StringBuilder sb = new StringBuilder(); - sb.append("/*\n" + - " * extra lines to offset the line number for the macro\n" + - " *\n" + - " *\n" + - " */\n" + - "\n" + - "\n" + - "define mymacro() returns void {\n" + - "A = load 'x' as ( u:int, v:long, w:bytearray);\n" + - "B = limit A 100;\n" + - "C = filter_typo B by 2 > 1;\n" + - "D = load 'y' as (d1, d2);\n" + - "E = join C by ( $0, $1 ), D by ( d1, d2 ) using 'replicated' parallel 16;\n" + - "F = store E into 'output';\n" + - "};\n" + - "mymacro();\n" - ); - - String expectedErr = - "/myscript.pig, line 11, column 0> Syntax error, unexpected symbol at or near 'C'"; - - validateFailure(sb.toString(), expectedErr, "/myscript.pig, line "); - } //see Pig-2184 @Test @@ -2279,135 +2252,6 @@ public class TestMacroExpansion { verify(script, expected); } - - // When declare-in-macro, macro param and command-line param contain the - // same name, last declare wins - @Test - public void testParamOverLap1() throws Exception { - String macro = - "DEFINE mygroupby(REL, key, number) RETURNS G {\n" + - " %declare number 333;\n" + - " $G = GROUP $REL by $key parallel $number;\n" + - "};"; - createFile("my_macro.pig", macro); - - String script = - "%declare number 111;\n" + - "IMPORT 'my_macro.pig';\n" + - "data = LOAD '1234.txt' USING PigStorage() AS (i: int);\n" + - "result = mygroupby(data, i, 222);\n" + - "STORE result INTO 'test.out' USING PigStorage();"; - - String expected = - "data = LOAD '1234.txt' USING PigStorage() AS i:int;\n" + - "result = GROUP data by (i) parallel 333;\n" + - "STORE result INTO 'test.out' USING PigStorage();\n"; - - verify(script, expected); - } - - // When default-in-macro, macro param and command-line param contain the - // same name, then default should be ignored and macro param to be taken - @Test - public void testParamOverLap2() throws Exception { - String macro = - "DEFINE mygroupby(REL, key, number) RETURNS G {\n" + - " %default number 333;\n" + - " $G = GROUP $REL by $key parallel $number;\n" + - "};"; - createFile("my_macro.pig", macro); - - String script = - "%declare number 111;\n" + - "IMPORT 'my_macro.pig';\n" + - "data = LOAD '1234.txt' USING PigStorage() AS (i: int);\n" + - "result = mygroupby(data, i, 222);\n" + - "STORE result INTO 'test.out' USING PigStorage();"; - - String expected = - "data = LOAD '1234.txt' USING PigStorage() AS i:int;\n" + - "result = GROUP data by (i) parallel 222;\n" + - "STORE result INTO 'test.out' USING PigStorage();\n"; - - verify(script, expected); - } - - // Overlapping of macro param and command-line param used to be disallowed. - // Now, simply taking the macro param when this happens - @Test - public void testParamOverLap3() throws Exception { - String macro = - "DEFINE mygroupby(REL, key, number) RETURNS G {\n" + - " $G = GROUP $REL by $key parallel $number;\n" + - "};"; - createFile("my_macro.pig", macro); - - String script = - "%default number 111;\n" + - "IMPORT 'my_macro.pig';\n" + - "data = LOAD '1234.txt' USING PigStorage() AS (i: int);\n" + - "result = mygroupby(data, i, 222);\n" + - "STORE result INTO 'test.out' USING PigStorage();"; - - String expected = - "data = LOAD '1234.txt' USING PigStorage() AS i:int;\n" + - "result = GROUP data by (i) parallel 222;\n" + - "STORE result INTO 'test.out' USING PigStorage();\n"; - - verify(script, expected); - } - - // Testing inline declare and commandline param overlap. - // testParamOverLap1 should cover this case as well but creating a specific - // case since this pair used to fail with NPE - @Test - public void testParamOverLap4() throws Exception { - String macro = - "DEFINE mygroupby(REL, key) RETURNS G {\n" + - " %declare number 333;\n" + - " $G = GROUP $REL by $key parallel $number;\n" + - "};"; - createFile("my_macro.pig", macro); - - String script = - "%default number 111;\n" + - "IMPORT 'my_macro.pig';\n" + - "data = LOAD '1234.txt' USING PigStorage() AS (i: int);\n" + - "result = mygroupby(data, i);\n" + - "STORE result INTO 'test.out' USING PigStorage();"; - - String expected = - "data = LOAD '1234.txt' USING PigStorage() AS i:int;\n" + - "result = GROUP data by (i) parallel 333;\n" + - "STORE result INTO 'test.out' USING PigStorage();\n"; - - verify(script, expected); - } - - // default-in-macro should yield to command-line param - @Test - public void testParamOverLap5() throws Exception { - String macro = - "DEFINE mygroupby(REL, key) RETURNS G {\n" + - " %default number 333;\n" + - " $G = GROUP $REL by $key parallel $number;\n" + - "};"; - createFile("my_macro.pig", macro); - - String script = - "%declare number 111;\n" + - "IMPORT 'my_macro.pig';\n" + - "data = LOAD '1234.txt' USING PigStorage() AS (i: int);\n" + - "result = mygroupby(data, i);\n" + - "STORE result INTO 'test.out' USING PigStorage();"; - - String expected = - "data = LOAD '1234.txt' USING PigStorage() AS i:int;\n" + - "result = GROUP data by (i) parallel 111;\n" + - "STORE result INTO 'test.out' USING PigStorage();\n"; - - verify(script, expected); - } //------------------------------------------------------------------------- Modified: pig/branches/spark/test/org/apache/pig/test/TestMultiQuery.java URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/test/TestMultiQuery.java?rev=1784224&r1=1784223&r2=1784224&view=diff ============================================================================== --- pig/branches/spark/test/org/apache/pig/test/TestMultiQuery.java (original) +++ pig/branches/spark/test/org/apache/pig/test/TestMultiQuery.java Fri Feb 24 03:34:37 2017 @@ -883,49 +883,6 @@ public class TestMultiQuery { } @Test - public void testMultiQueryJiraPig4883() throws Exception { - Storage.Data data = Storage.resetData(myPig); - data.set("inputLocation", - Storage.tuple("c", "12"), Storage.tuple("d", "-12")); - myPig.setBatchOn(); - myPig.registerQuery("A = load 'inputLocation' using mock.Storage();"); - myPig.registerQuery("A = foreach A generate (chararray)$0 as id, (long)$1 as val;"); - myPig.registerQuery("B = filter A by val > 0;"); - myPig.registerQuery("B1 = group B by val;"); - myPig.registerQuery("B1 = foreach B1 generate group as name, COUNT(B) as value;"); - myPig.registerQuery("B1 = foreach B1 generate (chararray)name,value;"); - myPig.registerQuery("store B1 into 'output1' using mock.Storage();"); - myPig.registerQuery("B2 = group B by id;"); - myPig.registerQuery("B2 = foreach B2 generate group as name, COUNT(B) as value;"); - myPig.registerQuery("store B2 into 'output2' using mock.Storage();"); - myPig.registerQuery("C = filter A by val < 0;"); - myPig.registerQuery("C1 = group C by val;"); - myPig.registerQuery("C1 = foreach C1 generate group as name, COUNT(C) as value;"); - myPig.registerQuery("store C1 into 'output3' using mock.Storage();"); - myPig.registerQuery("C2 = group C by id;"); - myPig.registerQuery("C2 = foreach C2 generate group as name, COUNT(C) as value;"); - myPig.registerQuery("store C2 into 'output4' using mock.Storage();"); - myPig.executeBatch(); - - List<Tuple> actualResults = data.get("output1"); - String[] expectedResults = new String[]{"(12, 1)"}; - Util.checkQueryOutputsAfterSortRecursive(actualResults.iterator(), expectedResults, org.apache.pig.newplan.logical.Util.translateSchema(myPig.dumpSchema("B1"))); - - - actualResults = data.get("output2"); - expectedResults = new String[]{"(c,1)"}; - Util.checkQueryOutputsAfterSortRecursive(actualResults.iterator(), expectedResults, org.apache.pig.newplan.logical.Util.translateSchema(myPig.dumpSchema("B2"))); - - actualResults = data.get("output3"); - expectedResults = new String[]{"(-12, 1)"}; - Util.checkQueryOutputsAfterSortRecursive(actualResults.iterator(), expectedResults, org.apache.pig.newplan.logical.Util.translateSchema(myPig.dumpSchema("C1"))); - - actualResults = data.get("output4"); - expectedResults = new String[]{"(d,1)"}; - Util.checkQueryOutputsAfterSortRecursive(actualResults.iterator(), expectedResults, org.apache.pig.newplan.logical.Util.translateSchema(myPig.dumpSchema("C2"))); - } - - @Test public void testMultiQueryJiraPig4899() throws Exception { myPig.setBatchOn(); Modified: pig/branches/spark/test/org/apache/pig/test/TestMultiQueryCompiler.java URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/test/TestMultiQueryCompiler.java?rev=1784224&r1=1784223&r2=1784224&view=diff ============================================================================== --- pig/branches/spark/test/org/apache/pig/test/TestMultiQueryCompiler.java (original) +++ pig/branches/spark/test/org/apache/pig/test/TestMultiQueryCompiler.java Fri Feb 24 03:34:37 2017 @@ -1558,7 +1558,14 @@ public class TestMultiQueryCompiler { MROperPlan mrp = null; try { - mrp = launcher.compile(pp, myPig.getPigContext()); + java.lang.reflect.Method compile = launcher.getClass() + .getDeclaredMethod("compile", + new Class[] { PhysicalPlan.class, PigContext.class }); + + compile.setAccessible(true); + + mrp = (MROperPlan) compile.invoke(launcher, new Object[] { pp, myPig.getPigContext() }); + Assert.assertNotNull(mrp); } catch (Exception e) { Modified: pig/branches/spark/test/org/apache/pig/test/TestNewPartitionFilterPushDown.java URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/test/TestNewPartitionFilterPushDown.java?rev=1784224&r1=1784223&r2=1784224&view=diff ============================================================================== --- pig/branches/spark/test/org/apache/pig/test/TestNewPartitionFilterPushDown.java (original) +++ pig/branches/spark/test/org/apache/pig/test/TestNewPartitionFilterPushDown.java Fri Feb 24 03:34:37 2017 @@ -57,8 +57,6 @@ import org.apache.pig.newplan.logical.ex import org.apache.pig.newplan.logical.expression.IsNullExpression; import org.apache.pig.newplan.logical.expression.LogicalExpression; import org.apache.pig.newplan.logical.expression.MapLookupExpression; -import org.apache.pig.newplan.logical.expression.NotEqualExpression; -import org.apache.pig.newplan.logical.expression.NotExpression; import org.apache.pig.newplan.logical.expression.OrExpression; import org.apache.pig.newplan.logical.expression.ProjectExpression; import org.apache.pig.newplan.logical.optimizer.LogicalPlanOptimizer; @@ -129,17 +127,6 @@ public class TestNewPartitionFilterPushD * @throws Exception */ @Test - public void testPartIsNullFilter() throws Exception { - String q = query + "b = filter a by srcid is null;" + "store b into 'out';"; - test(q, Arrays.asList("srcid"), - null, "(srcid is null)"); - } - - /** - * test case where filter only contains condition on partition cols - * @throws Exception - */ - @Test public void testOnlyPartFilter2() throws Exception { String q = query + "b = filter a by mrkt == 'us';" + "store b into 'out';"; test(q, Arrays.asList("srcid", "mrkt"), @@ -698,15 +685,6 @@ public class TestNewPartitionFilterPushD testFull(q, "((srcid < 5) or (srcid == 10))", "((f1 < 5) or (f2 == 'UK'))", false); } - // PIG-4940 - @Test - public void testUnaryExpressions() throws Exception { - String q = query + "b = filter a by srcid == 10 and not browser#'type' is null;" + - "store b into 'out';"; - test(q, Arrays.asList("srcid"), "(srcid == 10)", - "(not (browser#'type' is null))", true); - } - //// helper methods /////// private PartitionFilterExtractor test(String query, List<String> partitionCols, String expPartFilterString, String expFilterString) @@ -871,7 +849,7 @@ public class TestNewPartitionFilterPushD return "(" + input + ")"; } - public static String getTestExpression(LogicalExpression op) throws FrontendException { + private static String getTestExpression(LogicalExpression op) throws FrontendException { if(op == null) { return null; } @@ -893,8 +871,6 @@ public class TestNewPartitionFilterPushD opStr = " and "; } else if (op instanceof OrExpression) { opStr = " or "; - } else if (op instanceof NotEqualExpression) { - opStr = " != "; } else { opStr = op.getName(); } @@ -914,9 +890,6 @@ public class TestNewPartitionFilterPushD int colind = ((DereferenceExpression) op).getBagColumns().get(0); String column = String.valueOf(colind); return alias + ".$" + column; - } else if (op instanceof NotExpression) { - String expr = getTestExpression(((NotExpression) op).getExpression()); - return braketize("not " + expr); } else { throw new FrontendException("Unsupported conversion of LogicalExpression to Expression: " + op.getName()); } Modified: pig/branches/spark/test/org/apache/pig/test/TestNewPlanColumnPrune.java URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/test/TestNewPlanColumnPrune.java?rev=1784224&r1=1784223&r2=1784224&view=diff ============================================================================== --- pig/branches/spark/test/org/apache/pig/test/TestNewPlanColumnPrune.java (original) +++ pig/branches/spark/test/org/apache/pig/test/TestNewPlanColumnPrune.java Fri Feb 24 03:34:37 2017 @@ -454,34 +454,6 @@ public class TestNewPlanColumnPrune { } } - @Test - public void testNoAddForeach() throws Exception { - // PIG-5055 - // Need to make sure that it does not add foreach - // that drops all the fields from B2. - String query = "A = load 'd.txt' as (a0:int, a1:int, a2:int);" + - "B = load 'd.txt' as (b0:int, b1:int, b2:int);" + - "B2 = FILTER B by b0 == 0;" + - "C = join A by (1), B2 by (1) ;" + - "D = FOREACH C GENERATE A::a1, A::a2;" + - "store D into 'empty';"; - - LogicalPlan newLogicalPlan = buildPlan(query); - - PlanOptimizer optimizer = new MyPlanOptimizer(newLogicalPlan, 3); - optimizer.optimize(); - System.err.println(newLogicalPlan); - Iterator<Operator> iter = newLogicalPlan.getOperators(); - while (iter.hasNext()) { - Operator o = iter.next(); - LogicalRelationalOperator lro = (LogicalRelationalOperator)o; - if (lro == null || lro.getAlias() == null) continue; - if (lro.getAlias().equals("B2")) { - assertNotNull(lro.getSchema()); - } - } - } - public class MyPlanOptimizer extends LogicalPlanOptimizer { protected MyPlanOptimizer(OperatorPlan p, int iterations) { Modified: pig/branches/spark/test/org/apache/pig/test/TestPOGenerate.java URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/test/TestPOGenerate.java?rev=1784224&r1=1784223&r2=1784224&view=diff ============================================================================== --- pig/branches/spark/test/org/apache/pig/test/TestPOGenerate.java (original) +++ pig/branches/spark/test/org/apache/pig/test/TestPOGenerate.java Fri Feb 24 03:34:37 2017 @@ -21,10 +21,8 @@ package org.apache.pig.test; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; -import java.util.HashMap; import java.util.LinkedList; import java.util.List; -import java.util.Map; import java.util.Random; import org.apache.pig.backend.hadoop.executionengine.physicalLayer.POStatus; @@ -48,7 +46,6 @@ public class TestPOGenerate { DataBag cogroup; DataBag partialFlatten; DataBag simpleGenerate; - DataBag mapFlatten; Random r = new Random(); BagFactory bf = BagFactory.getInstance(); TupleFactory tf = TupleFactory.getInstance(); @@ -57,25 +54,10 @@ public class TestPOGenerate { public void setUp() throws Exception { Tuple [] inputA = new Tuple[4]; Tuple [] inputB = new Tuple[4]; - Tuple [] inputC = new Tuple[4]; for(int i = 0; i < 4; i++) { inputA[i] = tf.newTuple(2); inputB[i] = tf.newTuple(1); - inputC[i] = tf.newTuple(2); } - Map map0 = new HashMap<String,String>(); - Map map1 = new HashMap<String,String>(); - Map map2 = new HashMap<String,String>(); - Map map3 = new HashMap<String,String>(); - map0.put("A",""); - map0.put("B",""); - map1.put("A","a"); - map1.put("B","b"); - map2.put("A","aa"); - map2.put("B","bb"); - map3.put("A","aaa"); - map3.put("B","bbb"); - inputA[0].set(0, 'a'); inputA[0].set(1, '1'); inputA[1].set(0, 'b'); @@ -88,15 +70,6 @@ public class TestPOGenerate { inputB[1].set(0, 'b'); inputB[2].set(0, 'a'); inputB[3].set(0, 'd'); - inputC[0].set(0, 0); - inputC[0].set(1, map0); - inputC[1].set(0, 1); - inputC[1].set(1, map1); - inputC[2].set(0, 2); - inputC[2].set(1, map2); - inputC[3].set(0, 3); - inputC[3].set(1, map3); - DataBag cg11 = bf.newDefaultBag(); cg11.add(inputA[0]); cg11.add(inputA[2]); @@ -146,22 +119,15 @@ public class TestPOGenerate { tPartial[3].append(emptyBag); partialFlatten = bf.newDefaultBag(); - for (int i = 0; i < 4; ++i) { + for(int i = 0; i < 4; ++i) { partialFlatten.add(tPartial[i]); } simpleGenerate = bf.newDefaultBag(); - for (int i = 0; i < 4; ++i) { + for(int i = 0; i < 4; ++i) { simpleGenerate.add(inputA[i]); } - - mapFlatten = bf.newDefaultBag(); - for (int i = 0; i < inputC.length; ++i) { - mapFlatten.add(inputC[i]); - } - - //System.out.println("Cogroup : " + cogroup); //System.out.println("Partial : " + partialFlatten); //System.out.println("Simple : " + simpleGenerate); @@ -282,49 +248,4 @@ public class TestPOGenerate { assertEquals(simpleGenerate.size(), count); } - - @Test - public void testMapFlattenGenerate() throws Exception { - ExpressionOperator prj1 = new POProject(new OperatorKey("", r.nextLong()), -1, 0); - ExpressionOperator prj2 = new POProject(new OperatorKey("", r.nextLong()), -1, 1); - prj1.setResultType(DataType.INTEGER); - prj2.setResultType(DataType.MAP); - List<Boolean> toBeFlattened = new LinkedList<Boolean>(); - toBeFlattened.add(false); - toBeFlattened.add(true); - PhysicalPlan plan1 = new PhysicalPlan(); - plan1.add(prj1); - PhysicalPlan plan2 = new PhysicalPlan(); - plan2.add(prj2); - List<PhysicalPlan> inputs = new LinkedList<PhysicalPlan>(); - inputs.add(plan1); - inputs.add(plan2); - PhysicalOperator poGen = new POForEach(new OperatorKey("", r.nextLong()), 1, inputs, toBeFlattened); - - List<String> obtained = new LinkedList<String>(); - for (Tuple t : mapFlatten) { - poGen.attachInput(t); - Result output = poGen.getNextTuple(); - while(output.result != null && output.returnStatus != POStatus.STATUS_EOP) { - //System.out.println(output.result); - obtained.add(((Tuple) output.result).toString()); - output = poGen.getNextTuple(); - } - } - - int count = 0; - for (Tuple t : mapFlatten) { - Tuple expected = tf.newTuple(3); - expected.set(0, t.get(0)); - for (Object entryObj : ((Map)t.get(1)).entrySet()){ - Map.Entry entry = ((Map.Entry)entryObj); - expected.set(1, entry.getKey()); - expected.set(2, entry.getValue()); - assertTrue(obtained.contains(expected.toString())); - ++count; - } - } - assertEquals(mapFlatten.size()*2, count); - - } } Modified: pig/branches/spark/test/org/apache/pig/test/TestParamSubPreproc.java URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/test/TestParamSubPreproc.java?rev=1784224&r1=1784223&r2=1784224&view=diff ============================================================================== --- pig/branches/spark/test/org/apache/pig/test/TestParamSubPreproc.java (original) +++ pig/branches/spark/test/org/apache/pig/test/TestParamSubPreproc.java Fri Feb 24 03:34:37 2017 @@ -19,7 +19,6 @@ package org.apache.pig.test; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotEquals; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -43,7 +42,6 @@ import org.apache.pig.ExecType; import org.apache.pig.impl.PigContext; import org.apache.pig.tools.parameters.ParameterSubstitutionPreprocessor; import org.apache.pig.tools.parameters.ParseException; -import org.apache.pig.tools.pigstats.PigStats; import org.junit.Test; public class TestParamSubPreproc { @@ -54,9 +52,24 @@ public class TestParamSubPreproc { private FileInputStream pigExResultStream; private String basedir = "test/org/apache/pig/test/data"; - private void compareResults(InputStream expected, InputStream result) throws IOException { - BufferedReader inExpected = new BufferedReader(new InputStreamReader(expected)); - BufferedReader inResult = new BufferedReader(new InputStreamReader(result)); + /* Test case 1 + * Use a parameter within a pig script and provide value on the command line. + */ + @Test + public void testCmdlineParam() throws Exception{ + log.info("Starting test testCmdlineParam() ..."); + ParameterSubstitutionPreprocessor ps = new ParameterSubstitutionPreprocessor(50); + pigIStream = new BufferedReader(new FileReader(basedir + "/input1.pig")); + pigOStream = new FileWriter(basedir + "/output1.pig"); + + String[] arg = {"date=20080228"}; + String[] argFiles = null; + ps.genSubstitutedFile(pigIStream , pigOStream , arg , argFiles); + + FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig"); + pigExResultStream = new FileInputStream(basedir + "/ExpectedResult.pig"); + BufferedReader inExpected = new BufferedReader(new InputStreamReader(pigExResultStream)); + BufferedReader inResult = new BufferedReader(new InputStreamReader(pigResultStream)); String exLine; String resLine; @@ -76,26 +89,6 @@ public class TestParamSubPreproc { inExpected.close(); inResult.close(); - } - - /* Test case 1 - * Use a parameter within a pig script and provide value on the command line. - */ - @Test - public void testCmdlineParam() throws Exception{ - log.info("Starting test testCmdlineParam() ..."); - ParameterSubstitutionPreprocessor ps = new ParameterSubstitutionPreprocessor(50); - pigIStream = new BufferedReader(new FileReader(basedir + "/input1.pig")); - pigOStream = new FileWriter(basedir + "/output1.pig"); - - String[] arg = {"date=20080228"}; - String[] argFiles = null; - ps.genSubstitutedFile(pigIStream , pigOStream , arg , argFiles); - - FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig"); - pigExResultStream = new FileInputStream(basedir + "/ExpectedResult.pig"); - - compareResults(pigExResultStream, pigResultStream); log.info("Done"); @@ -117,8 +110,27 @@ public class TestParamSubPreproc { FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig"); pigExResultStream = new FileInputStream(basedir + "/ExpectedResult.pig"); + BufferedReader inExpected = new BufferedReader(new InputStreamReader(pigExResultStream)); + BufferedReader inResult = new BufferedReader(new InputStreamReader(pigResultStream)); + + String exLine; + String resLine; + int lineNum=0; + + while (true) { + lineNum++; + exLine = inExpected.readLine(); + resLine = inResult.readLine(); + if (exLine==null || resLine==null) + break; + assertEquals("Parameter substitution from config file failed. " + "Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum ,exLine.trim(), resLine.trim()); + } + if (!(exLine==null && resLine==null)) { + fail ("Parameter substitution from config file failed. " + "Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum); + } - compareResults(pigExResultStream, pigResultStream); + inExpected.close(); + inResult.close(); log.info("Done"); } @@ -140,8 +152,27 @@ public class TestParamSubPreproc { FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig"); pigExResultStream = new FileInputStream(basedir + "/ExpectedResultDefault.pig"); + BufferedReader inExpected = new BufferedReader(new InputStreamReader(pigExResultStream)); + BufferedReader inResult = new BufferedReader(new InputStreamReader(pigResultStream)); + + String exLine; + String resLine; + int lineNum=0; + + while (true) { + lineNum++; + exLine = inExpected.readLine(); + resLine = inResult.readLine(); + if (exLine==null || resLine==null) + break; + assertEquals("Parameter substitution with shell command failed. " + "Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum ,exLine.trim(), resLine.trim()); + } + if (!(exLine==null && resLine==null)) { + fail ("Parameter substitution with shell command failed. " + "Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum); + } - compareResults(pigExResultStream, pigResultStream); + inExpected.close(); + inResult.close(); log.info("Done"); } @@ -205,8 +236,27 @@ public class TestParamSubPreproc { FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig"); pigExResultStream = new FileInputStream(basedir + "/ExpectedResult4.pig"); + BufferedReader inExpected = new BufferedReader(new InputStreamReader(pigExResultStream)); + BufferedReader inResult = new BufferedReader(new InputStreamReader(pigResultStream)); + + String exLine; + String resLine; + int lineNum=0; + + while (true) { + lineNum++; + exLine = inExpected.readLine(); + resLine = inResult.readLine(); + if (exLine==null || resLine==null) + break; + assertEquals("Parameter substitution within a value failed. " + "Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum ,exLine.trim(), resLine.trim()); + } + if (!(exLine==null && resLine==null)) { + fail ("Parameter substitution within a value failed. " + "Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum); + } - compareResults(pigExResultStream, pigResultStream); + inExpected.close(); + inResult.close(); log.info("Done"); } @@ -230,9 +280,27 @@ public class TestParamSubPreproc { FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig"); pigExResultStream = new FileInputStream(basedir + "/ExpectedResult4.pig"); + BufferedReader inExpected = new BufferedReader(new InputStreamReader(pigExResultStream)); + BufferedReader inResult = new BufferedReader(new InputStreamReader(pigResultStream)); + + String exLine; + String resLine; + int lineNum=0; - compareResults(pigExResultStream, pigResultStream); + while (true) { + lineNum++; + exLine = inExpected.readLine(); + resLine = inResult.readLine(); + if (exLine==null || resLine==null) + break; + assertEquals("Parameter substitution with shell command failed. " + "Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum ,exLine.trim(), resLine.trim()); + } + if (!(exLine==null && resLine==null)) { + fail ("Parameter substitution with shell command failed. " + "Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum); + } + inExpected.close(); + inResult.close(); log.info("Done"); } @@ -253,8 +321,27 @@ public class TestParamSubPreproc { FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig"); pigExResultStream = new FileInputStream(basedir + "/ExpectedResultCmdLnPriorDeclare.pig"); + BufferedReader inExpected = new BufferedReader(new InputStreamReader(pigExResultStream)); + BufferedReader inResult = new BufferedReader(new InputStreamReader(pigResultStream)); + + String exLine; + String resLine; + int lineNum=0; + + while (true) { + lineNum++; + exLine = inExpected.readLine(); + resLine = inResult.readLine(); + if (exLine==null || resLine==null) + break; + assertEquals("Parameter substitution of command line arg. prior to declare stmt failed. " + "Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum ,exLine.trim(), resLine.trim()); + } + if (!(exLine==null && resLine==null)) { + fail ("Parameter substitution of command line arg. prior to declare stmt failed. " + "Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum); + } - compareResults(pigExResultStream, pigResultStream); + inExpected.close(); + inResult.close(); log.info("Done"); } @@ -277,9 +364,27 @@ public class TestParamSubPreproc { FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig"); pigExResultStream = new FileInputStream(basedir + "/ExpectedResult4.pig"); + BufferedReader inExpected = new BufferedReader(new InputStreamReader(pigExResultStream)); + BufferedReader inResult = new BufferedReader(new InputStreamReader(pigResultStream)); + + String exLine; + String resLine; + int lineNum=0; - compareResults(pigExResultStream, pigResultStream); + while (true) { + lineNum++; + exLine = inExpected.readLine(); + resLine = inResult.readLine(); + if (exLine==null || resLine==null) + break; + assertEquals("Parameter substitution for a command with shell command failed. " + "Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum ,exLine.trim(), resLine.trim()); + } + if (!(exLine==null && resLine==null)) { + fail ("Parameter substitution for a command with shell command failed. " + "Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum); + } + inExpected.close(); + inResult.close(); log.info("Done"); } @@ -301,9 +406,27 @@ public class TestParamSubPreproc { FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig"); pigExResultStream = new FileInputStream(basedir + "/ExpectedResult.pig"); + BufferedReader inExpected = new BufferedReader(new InputStreamReader(pigExResultStream)); + BufferedReader inResult = new BufferedReader(new InputStreamReader(pigResultStream)); + + String exLine; + String resLine; + int lineNum=0; - compareResults(pigExResultStream, pigResultStream); + while (true) { + lineNum++; + exLine = inExpected.readLine(); + resLine = inResult.readLine(); + if (exLine==null || resLine==null) + break; + assertEquals("Command line parameter substitution failed. " + "Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum ,exLine.trim(), resLine.trim()); + } + if (!(exLine==null && resLine==null)) { + fail ("Command line parameter substitution failed. " + "Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum); + } + inExpected.close(); + inResult.close(); log.info("Done"); } @@ -323,9 +446,27 @@ public class TestParamSubPreproc { FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig"); pigExResultStream = new FileInputStream(basedir + "/ExpectedResult.pig"); + BufferedReader inExpected = new BufferedReader(new InputStreamReader(pigExResultStream)); + BufferedReader inResult = new BufferedReader(new InputStreamReader(pigResultStream)); + + String exLine; + String resLine; + int lineNum=0; - compareResults(pigExResultStream, pigResultStream); + while (true) { + lineNum++; + exLine = inExpected.readLine(); + resLine = inResult.readLine(); + if (exLine==null || resLine==null) + break; + assertEquals("Parameter substitution from multiple config files failed. " + "Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum ,exLine.trim(), resLine.trim()); + } + if (!(exLine==null && resLine==null)) { + fail ("Parameter substitution from multiple config files failed. " + "Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum); + } + inExpected.close(); + inResult.close(); log.info("Done"); } @@ -346,9 +487,27 @@ public class TestParamSubPreproc { FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig"); pigExResultStream = new FileInputStream(basedir + "/ExpectedResult.pig"); + BufferedReader inExpected = new BufferedReader(new InputStreamReader(pigExResultStream)); + BufferedReader inResult = new BufferedReader(new InputStreamReader(pigResultStream)); + + String exLine; + String resLine; + int lineNum=0; - compareResults(pigExResultStream, pigResultStream); + while (true) { + lineNum++; + exLine = inExpected.readLine(); + resLine = inResult.readLine(); + if (exLine==null || resLine==null) + break; + assertEquals("Same Parameter substitution from multiple config files failed. " + "Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum ,exLine.trim(), resLine.trim()); + } + if (!(exLine==null && resLine==null)) { + fail ("Same Parameter substitution from multiple config files failed. " + "Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum); + } + inExpected.close(); + inResult.close(); log.info("Done"); } @@ -368,9 +527,27 @@ public class TestParamSubPreproc { FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig"); pigExResultStream = new FileInputStream(basedir + "/ExpectedResult.pig"); + BufferedReader inExpected = new BufferedReader(new InputStreamReader(pigExResultStream)); + BufferedReader inResult = new BufferedReader(new InputStreamReader(pigResultStream)); + + String exLine; + String resLine; + int lineNum=0; - compareResults(pigExResultStream, pigResultStream); + while (true) { + lineNum++; + exLine = inExpected.readLine(); + resLine = inResult.readLine(); + if (exLine==null || resLine==null) + break; + assertEquals("Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum ,exLine.trim(), resLine.trim()); + } + if (!(exLine==null && resLine==null)) { + fail ("Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum); + } + inExpected.close(); + inResult.close(); } /* Test case 15,16 @@ -438,8 +615,27 @@ public class TestParamSubPreproc { FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig"); pigExResultStream = new FileInputStream(basedir + "/ExpectedResult.pig"); + BufferedReader inExpected = new BufferedReader(new InputStreamReader(pigExResultStream)); + BufferedReader inResult = new BufferedReader(new InputStreamReader(pigResultStream)); - compareResults(pigExResultStream, pigResultStream); + String exLine; + String resLine; + int lineNum=0; + + while (true) { + lineNum++; + exLine = inExpected.readLine(); + resLine = inResult.readLine(); + if (exLine==null || resLine==null) + break; + assertEquals("Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum ,exLine.trim(), resLine.trim()); + } + if (!(exLine==null && resLine==null)) { + fail ("Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum); + } + + inExpected.close(); + inResult.close(); } /* Test case 21 @@ -458,8 +654,27 @@ public class TestParamSubPreproc { FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig"); pigExResultStream = new FileInputStream(basedir + "/ExpectedResult.pig"); + BufferedReader inExpected = new BufferedReader(new InputStreamReader(pigExResultStream)); + BufferedReader inResult = new BufferedReader(new InputStreamReader(pigResultStream)); - compareResults(pigExResultStream, pigResultStream); + String exLine; + String resLine; + int lineNum=0; + + while (true) { + lineNum++; + exLine = inExpected.readLine(); + resLine = inResult.readLine(); + if (exLine==null || resLine==null) + break; + assertEquals("Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum ,exLine.trim(), resLine.trim()); + } + if (!(exLine==null && resLine==null)) { + fail ("Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum); + } + + inExpected.close(); + inResult.close(); } @@ -478,9 +693,27 @@ public class TestParamSubPreproc { FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig"); pigExResultStream = new FileInputStream(basedir + "/ExpectedResult.pig"); + BufferedReader inExpected = new BufferedReader(new InputStreamReader(pigExResultStream)); + BufferedReader inResult = new BufferedReader(new InputStreamReader(pigResultStream)); - compareResults(pigExResultStream, pigResultStream); + String exLine; + String resLine; + int lineNum=0; + + while (true) { + lineNum++; + exLine = inExpected.readLine(); + resLine = inResult.readLine(); + if (exLine==null || resLine==null) + break; + assertEquals("Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum ,exLine.trim(), resLine.trim()); + } + if (!(exLine==null && resLine==null)) { + fail ("Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum); + } + inExpected.close(); + inResult.close(); } /* Test case 23 @@ -499,8 +732,27 @@ public class TestParamSubPreproc { FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig"); pigExResultStream = new FileInputStream(basedir + "/ExpectedResult.pig"); + BufferedReader inExpected = new BufferedReader(new InputStreamReader(pigExResultStream)); + BufferedReader inResult = new BufferedReader(new InputStreamReader(pigResultStream)); + + String exLine; + String resLine; + int lineNum=0; + + while (true) { + lineNum++; + exLine = inExpected.readLine(); + resLine = inResult.readLine(); + if (exLine==null || resLine==null) + break; + assertEquals("Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum ,exLine.trim(), resLine.trim()); + } + if (!(exLine==null && resLine==null)) { + fail ("Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum); + } - compareResults(pigExResultStream, pigResultStream); + inExpected.close(); + inResult.close(); } /* Test case 24 @@ -519,9 +771,28 @@ public class TestParamSubPreproc { FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig"); pigExResultStream = new FileInputStream(basedir + "/ExpectedResultMulDecs.pig"); + BufferedReader inExpected = new BufferedReader(new InputStreamReader(pigExResultStream)); + BufferedReader inResult = new BufferedReader(new InputStreamReader(pigResultStream)); - compareResults(pigExResultStream, pigResultStream); - } + String exLine; + String resLine; + int lineNum=0; + + while (true) { + lineNum++; + exLine = inExpected.readLine(); + resLine = inResult.readLine(); + if (exLine==null || resLine==null) + break; + assertEquals("Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum ,exLine.trim(), resLine.trim()); + } + if (!(exLine==null && resLine==null)) { + fail ("Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum); + } + + inExpected.close(); + inResult.close(); + } /* Test case 25 * Use %default to define param values @@ -538,8 +809,27 @@ public class TestParamSubPreproc { FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig"); pigExResultStream = new FileInputStream(basedir + "/ExpectedResult.pig"); + BufferedReader inExpected = new BufferedReader(new InputStreamReader(pigExResultStream)); + BufferedReader inResult = new BufferedReader(new InputStreamReader(pigResultStream)); + + String exLine; + String resLine; + int lineNum=0; - compareResults(pigExResultStream, pigResultStream); + while (true) { + lineNum++; + exLine = inExpected.readLine(); + resLine = inResult.readLine(); + if (exLine==null || resLine==null) + break; + assertEquals("Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum ,exLine.trim(), resLine.trim()); + } + if (!(exLine==null && resLine==null)) { + fail ("Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum); + } + + inExpected.close(); + inResult.close(); } /* Test case 26 @@ -557,8 +847,27 @@ public class TestParamSubPreproc { FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig"); pigExResultStream = new FileInputStream(basedir + "/ExpectedResult.pig"); + BufferedReader inExpected = new BufferedReader(new InputStreamReader(pigExResultStream)); + BufferedReader inResult = new BufferedReader(new InputStreamReader(pigResultStream)); + + String exLine; + String resLine; + int lineNum=0; + + while (true) { + lineNum++; + exLine = inExpected.readLine(); + resLine = inResult.readLine(); + if (exLine==null || resLine==null) + break; + assertEquals("Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum ,exLine.trim(), resLine.trim()); + } + if (!(exLine==null && resLine==null)) { + fail ("Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum); + } - compareResults(pigExResultStream, pigResultStream); + inExpected.close(); + inResult.close(); } @@ -577,8 +886,27 @@ public class TestParamSubPreproc { FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig"); pigExResultStream = new FileInputStream(basedir + "/ExpectedResult.pig"); + BufferedReader inExpected = new BufferedReader(new InputStreamReader(pigExResultStream)); + BufferedReader inResult = new BufferedReader(new InputStreamReader(pigResultStream)); + + String exLine; + String resLine; + int lineNum=0; + + while (true) { + lineNum++; + exLine = inExpected.readLine(); + resLine = inResult.readLine(); + if (exLine==null || resLine==null) + break; + assertEquals("Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum ,exLine.trim(), resLine.trim()); + } + if (!(exLine==null && resLine==null)) { + fail ("Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum); + } - compareResults(pigExResultStream, pigResultStream); + inExpected.close(); + inResult.close(); } /* Test case 29 @@ -596,8 +924,27 @@ public class TestParamSubPreproc { FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig"); pigExResultStream = new FileInputStream(basedir + "/ExpectedResult.pig"); + BufferedReader inExpected = new BufferedReader(new InputStreamReader(pigExResultStream)); + BufferedReader inResult = new BufferedReader(new InputStreamReader(pigResultStream)); + + String exLine; + String resLine; + int lineNum=0; - compareResults(pigExResultStream, pigResultStream); + while (true) { + lineNum++; + exLine = inExpected.readLine(); + resLine = inResult.readLine(); + if (exLine==null || resLine==null) + break; + assertEquals("Command line parameter substitution failed. " + "Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum ,exLine.trim(), resLine.trim()); + } + if (!(exLine==null && resLine==null)) { + fail ("Command line parameter substitution failed. " + "Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum); + } + + inExpected.close(); + inResult.close(); } /* Test case 30 @@ -615,8 +962,27 @@ public class TestParamSubPreproc { FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig"); pigExResultStream = new FileInputStream(basedir + "/ExpectedResult2.pig"); + BufferedReader inExpected = new BufferedReader(new InputStreamReader(pigExResultStream)); + BufferedReader inResult = new BufferedReader(new InputStreamReader(pigResultStream)); + + String exLine; + String resLine; + int lineNum=0; - compareResults(pigExResultStream, pigResultStream); + while (true) { + lineNum++; + exLine = inExpected.readLine(); + resLine = inResult.readLine(); + if (exLine==null || resLine==null) + break; + assertEquals("Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum ,exLine.trim(), resLine.trim()); + } + if (!(exLine==null && resLine==null)) { + fail ("Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum); + } + + inExpected.close(); + inResult.close(); } /* Test case 31 @@ -635,8 +1001,27 @@ public class TestParamSubPreproc { FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig"); pigExResultStream = new FileInputStream(basedir + "/ExpectedResult.pig"); + BufferedReader inExpected = new BufferedReader(new InputStreamReader(pigExResultStream)); + BufferedReader inResult = new BufferedReader(new InputStreamReader(pigResultStream)); + + String exLine; + String resLine; + int lineNum=0; + + while (true) { + lineNum++; + exLine = inExpected.readLine(); + resLine = inResult.readLine(); + if (exLine==null || resLine==null) + break; + assertEquals("Command line parameter substitution failed. " + "Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum ,exLine.trim(), resLine.trim()); + } + if (!(exLine==null && resLine==null)) { + fail ("Command line parameter substitution failed. " + "Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum); + } - compareResults(pigExResultStream, pigResultStream); + inExpected.close(); + inResult.close(); log.info("Done"); } @@ -657,9 +1042,27 @@ public class TestParamSubPreproc { FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig"); pigExResultStream = new FileInputStream(basedir + "/inputNoVars.pig"); + BufferedReader inExpected = new BufferedReader(new InputStreamReader(pigExResultStream)); + BufferedReader inResult = new BufferedReader(new InputStreamReader(pigResultStream)); + + String exLine; + String resLine; + int lineNum=0; - compareResults(pigExResultStream, pigResultStream); + while (true) { + lineNum++; + exLine = inExpected.readLine(); + resLine = inResult.readLine(); + if (exLine==null || resLine==null) + break; + assertEquals("Command line parameter substitution failed. " + "Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum ,exLine.trim(), resLine.trim()); + } + if (!(exLine==null && resLine==null)) { + fail ("Command line parameter substitution failed. " + "Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum); + } + inExpected.close(); + inResult.close(); log.info("Done"); } @@ -680,8 +1083,27 @@ public class TestParamSubPreproc { FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig"); pigExResultStream = new FileInputStream(basedir + "/ExpectedResult3.txt"); + BufferedReader inExpected = new BufferedReader(new InputStreamReader(pigExResultStream)); + BufferedReader inResult = new BufferedReader(new InputStreamReader(pigResultStream)); + + String exLine; + String resLine; + int lineNum=0; + + while (true) { + lineNum++; + exLine = inExpected.readLine(); + resLine = inResult.readLine(); + if (exLine==null || resLine==null) + break; + assertEquals("Command line parameter substitution failed. " + "Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum ,exLine.trim(), resLine.trim()); + } + if (!(exLine==null && resLine==null)) { + fail ("Command line parameter substitution failed. " + "Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum); + } - compareResults(pigExResultStream, pigResultStream); + inExpected.close(); + inResult.close(); log.info("Done"); } @@ -701,8 +1123,27 @@ public class TestParamSubPreproc { FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig"); pigExResultStream = new FileInputStream(basedir + "/ExpectedResultComment.pig"); + BufferedReader inExpected = new BufferedReader(new InputStreamReader(pigExResultStream)); + BufferedReader inResult = new BufferedReader(new InputStreamReader(pigResultStream)); - compareResults(pigExResultStream, pigResultStream); + String exLine; + String resLine; + int lineNum=0; + + while (true) { + lineNum++; + exLine = inExpected.readLine(); + resLine = inResult.readLine(); + if (exLine==null || resLine==null) + break; + assertEquals("Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum ,exLine.trim(), resLine.trim()); + } + if (!(exLine==null && resLine==null)) { + fail ("Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum); + } + + inExpected.close(); + inResult.close(); } /* Test case @@ -771,7 +1212,27 @@ public class TestParamSubPreproc { FileInputStream pigResultStream = new FileInputStream(basedir + "/output26.pig"); InputStream expected = new ByteArrayInputStream(expectedString.getBytes("UTF-8")); - compareResults(expected, pigResultStream); + BufferedReader inExpected = new BufferedReader(new InputStreamReader(expected)); + BufferedReader inResult = new BufferedReader(new InputStreamReader(pigResultStream)); + + String exLine; + String resLine; + int lineNum=0; + + while (true) { + lineNum++; + exLine = inExpected.readLine(); + resLine = inResult.readLine(); + if (exLine==null || resLine==null) + break; + assertEquals("Parameter substitution with shell command failed. " + "Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum ,exLine.trim(), resLine.trim()); + } + if (!(exLine==null && resLine==null)) { + fail ("Parameter substitution with shell command failed. " + "Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum); + } + + inExpected.close(); + inResult.close(); log.info("Done"); } @@ -791,8 +1252,27 @@ public class TestParamSubPreproc { FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig"); pigExResultStream = new FileInputStream(basedir + "/ExpectedResultDollarSign.pig"); + BufferedReader inExpected = new BufferedReader(new InputStreamReader(pigExResultStream)); + BufferedReader inResult = new BufferedReader(new InputStreamReader(pigResultStream)); + + String exLine; + String resLine; + int lineNum=0; + + while (true) { + lineNum++; + exLine = inExpected.readLine(); + resLine = inResult.readLine(); + if (exLine==null || resLine==null) + break; + assertEquals("Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum ,exLine.trim(), resLine.trim()); + } + if (!(exLine==null && resLine==null)) { + fail ("Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum); + } - compareResults(pigExResultStream, pigResultStream); + inExpected.close(); + inResult.close(); } @Test @@ -809,8 +1289,28 @@ public class TestParamSubPreproc { FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig"); pigExResultStream = new FileInputStream(basedir + "/ExpectedResult6.pig"); + BufferedReader inExpected = new BufferedReader(new InputStreamReader(pigExResultStream)); + BufferedReader inResult = new BufferedReader(new InputStreamReader(pigResultStream)); + + String exLine; + String resLine; + int lineNum=0; + + while (true) { + lineNum++; + exLine = inExpected.readLine(); + resLine = inResult.readLine(); + if (exLine==null || resLine==null) + break; + assertEquals("Command line parameter substitution failed. " + "Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum ,exLine.trim(), resLine.trim()); + } + if (!(exLine==null && resLine==null)) { + fail ("Command line parameter substitution failed. " + "Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum); + } + + inExpected.close(); + inResult.close(); - compareResults(pigExResultStream, pigResultStream); log.info("Done"); } @@ -826,9 +1326,27 @@ public class TestParamSubPreproc { FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig"); pigExResultStream = new FileInputStream(basedir + "/ExpectedResult7.pig"); + BufferedReader inExpected = new BufferedReader(new InputStreamReader(pigExResultStream)); + BufferedReader inResult = new BufferedReader(new InputStreamReader(pigResultStream)); - compareResults(pigExResultStream, pigResultStream); - log.info("Done"); + String exLine; + String resLine; + int lineNum=0; + + while (true) { + lineNum++; + exLine = inExpected.readLine(); + resLine = inResult.readLine(); + if (exLine==null || resLine==null) + break; + assertEquals("Command line parameter substitution failed. " + "Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum ,exLine.trim(), resLine.trim()); + } + if (!(exLine==null && resLine==null)) { + fail ("Command line parameter substitution failed. " + "Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum); + } + + inExpected.close(); + inResult.close(); } @Test @@ -880,351 +1398,6 @@ public class TestParamSubPreproc { assertEquals(resultContent, "daniel\t10\njenny\t20\n"); } - @Test - public void testCommandLineParamOverwritingDefault() throws Exception { - log.info("Starting test testCommandLineParamOverwritingDefault()"); - File inputFile = Util.createFile( - "runinput", - new String[] { "daniel\t10", - "jenny\t20;"}); - File output1 = File.createTempFile("output1_", ""); - output1.delete(); - - File script1 = Util.createFile("runscript1.pig", - new String[] { "%default output /invalidpathThatShouldFail;", - "a = load 'runinput';", - "store a into '$output';"}); - - PigStats stats = org.apache.pig.PigRunner.run(new String[] { - "-x", Util.getLocalTestMode().toString(), - "-p", "output=" + output1.getAbsolutePath(), - script1.getAbsolutePath()} , null); - Util.deleteDirectory(output1); - - assertTrue("job should succeed", stats.isSuccessful()); - assertTrue("Default param should be overridden by the commandline param", - output1.getAbsolutePath().endsWith(stats.getOutputNames().get(0))); - } - - @Test - public void testRunWithParamOverwritingDefault() throws Exception { - log.info("Starting test testScopeOfParamWithRunCommand()"); - File inputFile = Util.createFile( - "runinput", - new String[] { "daniel\t10", - "jenny\t20;"}); - File output1 = File.createTempFile("output1_", ""); - File output2 = File.createTempFile("output2_", ""); - output1.delete(); - output2.delete(); - - - File script1 = Util.createFile("runscript1.pig", - new String[] { "%default output '" + output2.getAbsolutePath() + "';", - "a = load 'runinput';", - "store a into '$output';"}); - - File mainscript = Util.createFile("mainscript.pig", - new String[] {"run -param output=" + output1.getAbsolutePath() - + " " + script1.getAbsolutePath() + ";"}); - - - PigStats stats = org.apache.pig.PigRunner.run(new String[] { - "-x", Util.getLocalTestMode().toString(), - mainscript.getAbsolutePath()} , null); - Util.deleteDirectory(output1); - Util.deleteDirectory(output2); - - assertTrue("job should succeed", stats.isSuccessful()); - assertEquals("There should only be 1 output.", - 1, stats.getOutputNames().size()); - assertEquals("Output name should be from output1 and not output2", - output1.getAbsolutePath(), - stats.getOutputLocations().get(0)); - } - - @Test - public void testScopeOfParamWithRunCommand() throws Exception { - log.info("Starting test testScopeOfParamWithRunCommand()"); - File inputFile = Util.createFile( - "runinput", - new String[] { "daniel\t10", - "jenny\t20;"}); - File output1 = File.createTempFile("output1_", ""); - File output2 = File.createTempFile("output2_", ""); - output1.delete(); - output2.delete(); - - File script1 = Util.createFile("runscript1.pig", - new String[] { "%default output '" + output1.getAbsolutePath() + "';", - "a = load 'runinput';", - "store a into '$output';"}); - - File script2 = Util.createFile("runscript2.pig", - new String[] { "%default output '" + output2.getAbsolutePath() + "';", - "a = load 'runinput';", - "store a into '$output';"}); - - File mainscript = Util.createFile("mainscript.pig", - new String[] { "run " + script1.getAbsolutePath() + ";", - "run " + script2.getAbsolutePath() + ";" }); - - PigStats stats = org.apache.pig.PigRunner.run(new String[] { - "-x", Util.getLocalTestMode().toString(), - mainscript.getAbsolutePath()} , null); - Util.deleteDirectory(output1); - Util.deleteDirectory(output2); - - assertTrue("job should succeed", stats.isSuccessful()); - assertNotEquals("Two output paths should differ", - stats.getOutputNames().get(0), stats.getOutputNames().get(1)); - assertEquals("Each output should contain 2 records", - 2, stats.getOutputStats().get(0).getNumberRecords()); - assertEquals("Each output should contain 2 records", - 2, stats.getOutputStats().get(1).getNumberRecords()); - } - - @Test - public void testScopeOfParamWithNestedRunCommand() throws Exception { - log.info("Starting test testScopeOfParamWithRunCommand()"); - File inputFile = Util.createFile( - "runinput", - new String[] { "daniel\t10", - "jenny\t20;"}); - /* - * script1 sets a=1, b=2, c=3; calls script2 - * script2 sets b=22 (by -param); calls script3 - * script3 sets c=333; saves $a$b$c (122333) - * script2 saves $a$b$c (1223) - * script1 saves $a$b$c (123) - */ - File script3 = Util.createFile("runscript3.pig", - new String[] { "%declare c '333';", - "a = load 'runinput';", - "store a into 'testScopeOfParamWithNestedRunCommand${a}${b}${c}';"}); - - File script2 = Util.createFile("runscript2.pig", - new String[] { "run " + script3.getAbsolutePath() + ";", - "a = load 'runinput';", - "store a into 'testScopeOfParamWithNestedRunCommand${a}${b}${c}';"}); - - File script1 = Util.createFile("runscript1.pig", - new String[] { "%declare a '1';", - "%declare b '2';", - "%declare c '3';", - "run -param b=22 " + script2.getAbsolutePath() + ";", - "a = load 'runinput';", - "store a into 'testScopeOfParamWithNestedRunCommand${a}${b}${c}';"}); - - PigStats stats = org.apache.pig.PigRunner.run(new String[] { - "-x", Util.getLocalTestMode().toString(), - script1.getAbsolutePath()} , null); - - for( String output : stats.getOutputNames() ) { - assertTrue(output.contains("testScopeOfParamWithNestedRunCommand")); - Util.deleteDirectory(new File(output)); - } - assertTrue("job should succeed", stats.isSuccessful()); - assertEquals("There should be three outputs.", 3, stats.getOutputNames().size()); - - for( String expectedoutput : new String [] {"testScopeOfParamWithNestedRunCommand123", - "testScopeOfParamWithNestedRunCommand1223", - "testScopeOfParamWithNestedRunCommand122333"} ) { - boolean found=false; - for( String output : stats.getOutputNames() ) { - if( output.endsWith(expectedoutput) ) { - found=true; - } - } - assertTrue("Output " + expectedoutput + " should exist.", found); - } - } - - /* This currently does not work since PigMacro only picks the - * param setting from the root script (script1) - * To revisit after Grunt moves to ANTLR in PIG-2597. - * Tracking in PIG-5028. - * - - @Test - public void testScopeOfParamWithMacro() throws Exception { - log.info("Starting test testScopeOfParamWithMacro()"); - File inputFile = Util.createFile( - "runinput", - new String[] { "daniel\t10", - "jenny\t20;"}); - File macro = Util.createFile("testmacro.pig", - new String[] { "DEFINE mymacro (A) RETURNS void {", - "store $A into 'testScopeOfParamWithMacro${a}${b}${c}';", - "};"}); - - File script3 = Util.createFile("runscript3.pig", - new String[] { "%declare c '333';"}); - - File script2 = Util.createFile("runscript2.pig", - new String[] { "%declare b '22';", - "import '" + macro.getAbsolutePath() + "';", - "a = load 'runinput';", - "mymacro(a);", - "exec " + script3.getAbsolutePath() + ";"}); - - File script1 = Util.createFile("runscript1.pig", - new String[] { "%declare a '1';", - "%declare b '2';", - "%declare c '3';", - "exec " + script2.getAbsolutePath() + ";"}); - - PigStats stats = org.apache.pig.PigRunner.run(new String[] { - "-x", Util.getLocalTestMode().toString(), - script1.getAbsolutePath()} , null); - - assertTrue("job should succeed", stats.isSuccessful()); - Util.deleteDirectory(new File(stats.getOutputNames().get(0))); - assertEquals("There should be only 1 output.", 1, stats.getOutputNames().size()); - assertTrue("Expected output testScopeOfParamWithMacro1223 but got " + stats.getOutputNames().get(0), - stats.getOutputNames().get(0).equals("testScopeOfParamWithMacro1223")); - } - */ - - - /* - * Test parameter substition when register contains /* globbing - */ - @Test - public void testSubstitutionWithRegisterGlobbing() throws Exception{ - log.info("Starting test testSubstitutionWithRegisterGlobbing()"); - final String queryString = - "register /abc/$regdir/*.jar;\n" + - "A = LOAD '$input' USING PigStorage ();\n" + - "STORE A INTO '$output';\n" + - " /* comment that would make register globbing to be part of the multi-line comment */\n"; - - - ParameterSubstitutionPreprocessor ps = new ParameterSubstitutionPreprocessor(50); - pigIStream = new BufferedReader( - new InputStreamReader(new ByteArrayInputStream(queryString.getBytes("UTF-8")))); - pigOStream = new FileWriter(basedir + "/output1.pig"); - - String[] arg = {"input = 'input.txt'", "output = 'output.txt'", "regdir = 'def'"}; - String[] argFiles = null; - ps.genSubstitutedFile(pigIStream , pigOStream , arg , argFiles); - - FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig"); - - String expectedString = queryString.replaceAll("\\$input","input.txt") - .replaceAll("\\$output","output.txt") - .replaceAll("\\$regdir","def"); - InputStream expected = new ByteArrayInputStream(expectedString.getBytes("UTF-8")); - - compareResults(expected, pigResultStream); - - log.info("Done"); - } - - /* - * Test parameter substition when load contains /* globbing - */ - @Test - public void testSubstitutionWithLoadGlobbing() throws Exception{ - log.info("Starting test testSubstitutionWithLoadGlobbing()"); - final String queryString = - "A = LOAD '/zzz/*' USING PigStorage ();\n" + - "STORE A INTO '$output';\n" + - " /* comment that would make register globbing to be part of the multi-line comment */\n"; - - - ParameterSubstitutionPreprocessor ps = new ParameterSubstitutionPreprocessor(50); - pigIStream = new BufferedReader( - new InputStreamReader(new ByteArrayInputStream(queryString.getBytes("UTF-8")))); - pigOStream = new FileWriter(basedir + "/output1.pig"); - - String[] arg = {"output = 'output.txt'"}; - String[] argFiles = null; - ps.genSubstitutedFile(pigIStream , pigOStream , arg , argFiles); - - FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig"); - - String expectedString = queryString.replaceAll("\\$output","output.txt"); - InputStream expected = new ByteArrayInputStream(expectedString.getBytes("UTF-8")); - - compareResults(expected, pigResultStream); - - log.info("Done"); - } - - @Test - public void testSubstitutionWithRedeclaration() throws Exception{ - log.info("Starting test testSubstitutionWithRedeclaration()"); - final String queryString = - "%declare output '/tmp/abc';\n" + - "%declare actualoutput '$output.out';\n" + - "A = load 'input.txt' ;\n" + - "store A into '$actualoutput';\n" + - "%declare output '/tmp/def';\n" + - "%declare actualoutput '$output.out';\n" + - "store A into '$actualoutput';"; - - - ParameterSubstitutionPreprocessor ps = new ParameterSubstitutionPreprocessor(50); - pigIStream = new BufferedReader( - new InputStreamReader(new ByteArrayInputStream(queryString.getBytes("UTF-8")))); - pigOStream = new FileWriter(basedir + "/output1.pig"); - - String[] arg = {"output = 'output.txt'"}; - String[] argFiles = null; - ps.genSubstitutedFile(pigIStream , pigOStream , arg , argFiles); - - FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig"); - - String expectedString = queryString.replaceAll("%declare [0-9a-zA-Z.'/\\$; ]*\n",";\n") - .replaceAll("\\$","") - .replaceFirst("actualoutput","/tmp/abc.out") - .replaceFirst("actualoutput","/tmp/def.out"); - InputStream expected = new ByteArrayInputStream(expectedString.getBytes("UTF-8")); - - compareResults(expected, pigResultStream); - - log.info("Done"); - } - - @Test - public void testSubstitutionWithRedeclaredShell() throws Exception{ - log.info("Starting test testSubstitutionWithRedeclaredShell()"); - final String queryString = - "A = load 'input.txt' ;\n" + - "%declare now `bash -c \"date +'%Y%m%d_%H:%M:%S'; sleep 1;\"`;\n" + - "store A into '$now';\n" + - "%declare now `bash -c \"date +'%Y%m%d_%H:%M:%S'; sleep 1;\"`;\n" + - "store A into '$now';\n"; - - ParameterSubstitutionPreprocessor ps = new ParameterSubstitutionPreprocessor(50); - pigIStream = new BufferedReader( - new InputStreamReader(new ByteArrayInputStream(queryString.getBytes("UTF-8")))); - pigOStream = new FileWriter(basedir + "/output1.pig"); - - String[] arg = {"output = 'output.txt'"}; - String[] argFiles = null; - ps.genSubstitutedFile(pigIStream , pigOStream , arg , argFiles); - - BufferedReader pigresult = new BufferedReader(new InputStreamReader(new FileInputStream(basedir + "/output1.pig"))); - - - String [] filenames = new String [2]; - int index=0; - String line; - while ((line = pigresult.readLine())!=null) { - if( line.startsWith("store A into") ) { - filenames[index++] = line.split(" ")[3]; - } - } - - assertEquals("There should be 2 store statements", 2, index); - assertNotEquals("Identical shell param should be reexecuted.", - filenames[0], - filenames[1]); - log.info("Done"); - } - @SuppressWarnings("resource") private BufferedReader WithConditionalReplacement(String filename, String orig, String dest, boolean replace) throws IOException { BufferedReader pigOrigIStream = new BufferedReader(new FileReader(filename)); Modified: pig/branches/spark/test/org/apache/pig/test/TestPigContext.java URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/test/TestPigContext.java?rev=1784224&r1=1784223&r2=1784224&view=diff ============================================================================== --- pig/branches/spark/test/org/apache/pig/test/TestPigContext.java (original) +++ pig/branches/spark/test/org/apache/pig/test/TestPigContext.java Fri Feb 24 03:34:37 2017 @@ -27,7 +27,6 @@ import java.util.List; import java.util.Properties; import java.util.Random; -import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.mapred.FileAlreadyExistsException; import org.apache.pig.PigServer; import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MRConfiguration; @@ -271,7 +270,7 @@ public class TestPigContext { assertEquals(JOB_TRACKER, pigServer.getPigContext().getProperties().getProperty(MRConfiguration.JOB_TRACKER)); assertEquals(FS_NAME, - pigServer.getPigContext().getProperties().getProperty(FileSystem.FS_DEFAULT_NAME_KEY)); + pigServer.getPigContext().getProperties().getProperty("fs.default.name")); assertEquals(TMP_DIR_PROP, pigServer.getPigContext().getProperties().getProperty("hadoop.tmp.dir")); }
