Modified: pig/branches/spark/test/org/apache/pig/builtin/TestAvroStorage.java URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/builtin/TestAvroStorage.java?rev=1654955&r1=1654954&r2=1654955&view=diff ============================================================================== --- pig/branches/spark/test/org/apache/pig/builtin/TestAvroStorage.java (original) +++ pig/branches/spark/test/org/apache/pig/builtin/TestAvroStorage.java Tue Jan 27 02:27:45 2015 @@ -16,6 +16,25 @@ */ package org.apache.pig.builtin; +import static org.apache.pig.builtin.mock.Storage.resetData; +import static org.apache.pig.builtin.mock.Storage.tuple; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.io.PrintStream; +import java.io.PrintWriter; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; +import java.util.TreeSet; + import org.apache.avro.file.DataFileStream; import org.apache.avro.generic.GenericData; import org.apache.avro.generic.GenericData.Record; @@ -51,25 +70,6 @@ import org.junit.Test; import com.google.common.collect.ImmutableMap; -import java.io.File; -import java.io.FileWriter; -import java.io.IOException; -import java.io.PrintStream; -import java.io.PrintWriter; -import java.util.ArrayList; -import java.util.Comparator; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.TreeMap; -import java.util.TreeSet; - -import static junit.framework.Assert.assertEquals; -import static org.junit.Assert.assertTrue; -import static org.apache.pig.builtin.mock.Storage.resetData; -import static org.apache.pig.builtin.mock.Storage.tuple; - public class TestAvroStorage { final protected static Log LOG = LogFactory.getLog(TestAvroStorage.class); @@ -103,6 +103,7 @@ public class TestAvroStorage { "recordsWithFixed", "recordsWithMaps", "recordsWithMapsOfRecords", + "recordsWithMapsOfArrayOfRecords", "recordsWithNullableUnions", "recordWithRepeatedSubRecords", "recursiveRecord", @@ -118,6 +119,7 @@ public class TestAvroStorage { private static PigServer pigServerLocal = null; public static final PathFilter hiddenPathFilter = new PathFilter() { + @Override public boolean accept(Path p) { String name = p.getName(); return !name.startsWith("_") && !name.startsWith("."); @@ -324,9 +326,9 @@ public class TestAvroStorage { return outbasedir + st[index].getMethodName(); } - @Test public void testLoadRecordsOfStringArrays() throws Exception { + @Test + public void testLoadRecordsOfStringArrays() throws Exception { final String input = basedir + "data/avro/uncompressed/recordsOfStringArrays.avro"; - final String check = input; testAvroStorage(true, basedir + "code/pig/dump.pig", ImmutableMap.of( "INFILE", input, @@ -336,7 +338,8 @@ public class TestAvroStorage { } - @Test public void testLoadRecords() throws Exception { + @Test + public void testLoadRecords() throws Exception { final String input = basedir + "data/avro/uncompressed/records.avro"; final String check = basedir + "data/avro/uncompressed/recordsAsOutputByPig.avro"; testAvroStorage(true, basedir + "code/pig/identity_ao2.pig", @@ -349,7 +352,8 @@ public class TestAvroStorage { verifyResults(createOutputName(),check); } - @Test public void testLoadRecordsWithSimpleUnion() throws Exception { + @Test + public void testLoadRecordsWithSimpleUnion() throws Exception { final String input = basedir + "data/avro/uncompressed/recordsWithSimpleUnion.avro"; final String check = basedir + "data/avro/uncompressed/recordsWithSimpleUnionOutput.avro"; testAvroStorage(true, basedir + "code/pig/identity_ao2.pig", @@ -362,7 +366,8 @@ public class TestAvroStorage { verifyResults(createOutputName(),check); } - @Test public void testProjection() throws Exception { + @Test + public void testProjection() throws Exception { final String input = basedir + "data/avro/uncompressed/records.avro"; final String check = basedir + "data/avro/uncompressed/projectionTest.avro"; testAvroStorage(true, basedir + "code/pig/projection_test.pig", @@ -375,7 +380,8 @@ public class TestAvroStorage { verifyResults(createOutputName(),check); } - @Test public void testProjectionWithSchema() throws Exception { + @Test + public void testProjectionWithSchema() throws Exception { final String input = basedir + "data/avro/uncompressed/records.avro"; final String check = basedir + "data/avro/uncompressed/projectionTestWithSchema.avro"; testAvroStorage(true, basedir + "code/pig/projection_test_with_schema.pig", @@ -389,7 +395,8 @@ public class TestAvroStorage { verifyResults(createOutputName(),check); } - @Test public void testDates() throws Exception { + @Test + public void testDates() throws Exception { final String input = basedir + "data/avro/uncompressed/records.avro"; final String check = basedir + "data/avro/uncompressed/recordsAsOutputByPigWithDates.avro"; testAvroStorage(true, basedir + "code/pig/with_dates.pig", @@ -402,7 +409,8 @@ public class TestAvroStorage { verifyResults(createOutputName(),check); } - @Test public void testLoadRecordsSpecifyFullSchema() throws Exception { + @Test + public void testLoadRecordsSpecifyFullSchema() throws Exception { final String input = basedir + "data/avro/uncompressed/records.avro"; final String check = basedir + "data/avro/uncompressed/recordsAsOutputByPig.avro"; final String schema = loadFileIntoString(basedir + "schema/records.avsc"); @@ -417,7 +425,8 @@ public class TestAvroStorage { verifyResults(createOutputName(),check); } - @Test public void testLoadRecordsSpecifyFullSchemaFromFile() throws Exception { + @Test + public void testLoadRecordsSpecifyFullSchemaFromFile() throws Exception { final String input = basedir + "data/avro/uncompressed/records.avro"; final String check = basedir + "data/avro/uncompressed/recordsAsOutputByPig.avro"; testAvroStorage(true, basedir + "code/pig/identity.pig", @@ -431,7 +440,8 @@ public class TestAvroStorage { verifyResults(createOutputName(),check); } - @Test public void testLoadRecordsSpecifySubSchema() throws Exception { + @Test + public void testLoadRecordsSpecifySubSchema() throws Exception { final String input = basedir + "data/avro/uncompressed/records.avro"; final String check = basedir + "data/avro/uncompressed/recordsSubSchema.avro"; testAvroStorage(true, basedir + "code/pig/identity_ai1_ao2.pig", @@ -445,7 +455,8 @@ public class TestAvroStorage { verifyResults(createOutputName(),check); } - @Test public void testLoadRecordsSpecifySubSchemaFromFile() throws Exception { + @Test + public void testLoadRecordsSpecifySubSchemaFromFile() throws Exception { final String input = basedir + "data/avro/uncompressed/records.avro"; final String check = basedir + "data/avro/uncompressed/recordsSubSchema.avro"; testAvroStorage(true, basedir + "code/pig/identity_blank_first_args.pig", @@ -458,7 +469,8 @@ public class TestAvroStorage { verifyResults(createOutputName(),check); } - @Test public void testLoadRecordsSpecifySubSchemaFromExampleFile() throws Exception { + @Test + public void testLoadRecordsSpecifySubSchemaFromExampleFile() throws Exception { final String input = basedir + "data/avro/uncompressed/records.avro"; final String check = basedir + "data/avro/uncompressed/recordsSubSchema.avro"; testAvroStorage(true, basedir + "code/pig/identity_blank_first_args.pig", @@ -471,7 +483,8 @@ public class TestAvroStorage { verifyResults(createOutputName(),check); } - @Test public void testLoadRecordsOfArrays() throws Exception { + @Test + public void testLoadRecordsOfArrays() throws Exception { final String input = basedir + "data/avro/uncompressed/recordsOfArrays.avro"; final String check = input; testAvroStorage(true, basedir + "code/pig/identity_just_ao2.pig", @@ -483,7 +496,8 @@ public class TestAvroStorage { verifyResults(createOutputName(),check); } - @Test public void testLoadRecordsOfArraysOfRecords() throws Exception { + @Test + public void testLoadRecordsOfArraysOfRecords() throws Exception { final String input = basedir + "data/avro/uncompressed/recordsOfArraysOfRecords.avro"; final String check = input; testAvroStorage(true, basedir + "code/pig/identity_just_ao2.pig", @@ -495,7 +509,8 @@ public class TestAvroStorage { verifyResults(createOutputName(),check); } - @Test public void testLoadRecordsWithEnums() throws Exception { + @Test + public void testLoadRecordsWithEnums() throws Exception { final String input = basedir + "data/avro/uncompressed/recordsWithEnums.avro"; final String check = input; testAvroStorage(true, basedir + "code/pig/identity_just_ao2.pig", @@ -507,7 +522,8 @@ public class TestAvroStorage { verifyResults(createOutputName(),check); } - @Test public void testLoadRecordsWithFixed() throws Exception { + @Test + public void testLoadRecordsWithFixed() throws Exception { final String input = basedir + "data/avro/uncompressed/recordsWithFixed.avro"; final String check = input; testAvroStorage(true, basedir + "code/pig/identity_just_ao2.pig", @@ -519,7 +535,8 @@ public class TestAvroStorage { verifyResults(createOutputName(),check); } - @Test public void testLoadRecordsWithMaps() throws Exception { + @Test + public void testLoadRecordsWithMaps() throws Exception { final String input = basedir + "data/avro/uncompressed/recordsWithMaps.avro"; final String check = input; testAvroStorage(true, basedir + "code/pig/identity_just_ao2.pig", @@ -531,7 +548,8 @@ public class TestAvroStorage { verifyResults(createOutputName(),check); } - @Test public void testLoadRecordsWithMapsOfRecords() throws Exception { + @Test + public void testLoadRecordsWithMapsOfRecords() throws Exception { final String input = basedir + "data/avro/uncompressed/recordsWithMapsOfRecords.avro"; final String check = input; testAvroStorage(true, basedir + "code/pig/identity_just_ao2.pig", @@ -543,7 +561,21 @@ public class TestAvroStorage { verifyResults(createOutputName(),check); } - @Test public void testLoadRecordsWithNullableUnions() throws Exception { + @Test + public void testLoadRecordsWithMapsOfArrayOfRecords() throws Exception { + final String input = basedir + "data/avro/uncompressed/recordsWithMapsOfArrayOfRecords.avro"; + final String check = input; + testAvroStorage(true, basedir + "code/pig/identity_just_ao2.pig", + ImmutableMap.of( + "INFILE", input, + "AVROSTORAGE_OUT_2", "-f " + basedir + "schema/recordsWithMapsOfArrayOfRecords.avsc", + "OUTFILE", createOutputName()) + ); + verifyResults(createOutputName(),check); + } + + @Test + public void testLoadRecordsWithNullableUnions() throws Exception { final String input = basedir + "data/avro/uncompressed/recordsWithNullableUnions.avro"; final String check = input; testAvroStorage(true, basedir + "code/pig/identity_just_ao2.pig", @@ -555,7 +587,8 @@ public class TestAvroStorage { verifyResults(createOutputName(),check); } - @Test public void testLoadDeflateCompressedRecords() throws Exception { + @Test + public void testLoadDeflateCompressedRecords() throws Exception { final String input = basedir + "data/avro/compressed/deflate/records.avro"; final String check = basedir + "data/avro/uncompressed/recordsAsOutputByPig.avro"; testAvroStorage(true, basedir + "code/pig/identity_ao2.pig", @@ -568,7 +601,8 @@ public class TestAvroStorage { verifyResults(createOutputName(),check); } - @Test public void testLoadSnappyCompressedRecords() throws Exception { + @Test + public void testLoadSnappyCompressedRecords() throws Exception { final String input = basedir + "data/avro/compressed/snappy/records.avro"; final String check = basedir + "data/avro/uncompressed/recordsAsOutputByPig.avro"; testAvroStorage(true, basedir + "code/pig/identity_ao2.pig", @@ -581,7 +615,8 @@ public class TestAvroStorage { verifyResults(createOutputName(),check); } - @Test public void testStoreDeflateCompressedRecords() throws Exception { + @Test + public void testStoreDeflateCompressedRecords() throws Exception { final String input = basedir + "data/avro/uncompressed/records.avro"; final String check = basedir + "data/avro/compressed/deflate/recordsAsOutputByPig.avro"; testAvroStorage(true, basedir + "code/pig/identity_codec.pig", @@ -597,7 +632,8 @@ public class TestAvroStorage { verifyResults(createOutputName(),check); } - @Test public void testStoreSnappyCompressedRecords() throws Exception { + @Test + public void testStoreSnappyCompressedRecords() throws Exception { final String input = basedir + "data/avro/uncompressed/records.avro"; final String check = basedir + "data/avro/compressed/snappy/recordsAsOutputByPig.avro"; testAvroStorage(true, basedir + "code/pig/identity_codec.pig", @@ -613,7 +649,8 @@ public class TestAvroStorage { verifyResults(createOutputName(),check); } - @Test public void testLoadRecursiveRecords() throws Exception { + @Test + public void testLoadRecursiveRecords() throws Exception { final String input = basedir + "data/avro/uncompressed/recursiveRecord.avro"; testAvroStorage(false, basedir + "code/pig/recursive_tests.pig", ImmutableMap.of( @@ -625,7 +662,8 @@ public class TestAvroStorage { ); } - @Test public void testLoadRecursiveRecordsOptionOn() throws Exception { + @Test + public void testLoadRecursiveRecordsOptionOn() throws Exception { final String input = basedir + "data/avro/uncompressed/recursiveRecord.avro"; final String check = basedir + "data/avro/uncompressed/recordsSubSchemaNullable.avro"; testAvroStorage(true, basedir + "code/pig/recursive_tests.pig", @@ -639,7 +677,8 @@ public class TestAvroStorage { verifyResults(createOutputName(),check); } - @Test public void testLoadRecordsWithRepeatedSubRecords() throws Exception { + @Test + public void testLoadRecordsWithRepeatedSubRecords() throws Exception { final String input = basedir + "data/avro/uncompressed/recordWithRepeatedSubRecords.avro"; final String check = basedir + "data/avro/uncompressed/recordWithRepeatedSubRecords.avro"; testAvroStorage(true, basedir + "code/pig/identity_just_ao2.pig", @@ -651,7 +690,8 @@ public class TestAvroStorage { verifyResults(createOutputName(),check); } - @Test public void testLoadDirectory() throws Exception { + @Test + public void testLoadDirectory() throws Exception { final String input = basedir + "data/avro/uncompressed/testdirectory"; final String check = basedir + "data/avro/uncompressed/testDirectoryCounts.avro"; testAvroStorage(true, basedir + "code/pig/directory_test.pig", @@ -664,7 +704,8 @@ public class TestAvroStorage { verifyResults(createOutputName(),check); } - @Test public void testLoadGlob() throws Exception { + @Test + public void testLoadGlob() throws Exception { final String input = basedir + "data/avro/uncompressed/testdirectory/part-m-0000*"; final String check = basedir + "data/avro/uncompressed/testDirectoryCounts.avro"; testAvroStorage(true, basedir + "code/pig/directory_test.pig", @@ -677,7 +718,8 @@ public class TestAvroStorage { verifyResults(createOutputName(),check); } - @Test public void testPartialLoadGlob() throws Exception { + @Test + public void testPartialLoadGlob() throws Exception { final String input = basedir + "data/avro/uncompressed/testdirectory/part-m-0000{0,2,4,6}.avro"; final String check = basedir + "data/avro/uncompressed/evenFileNameTestDirectoryCounts.avro"; testAvroStorage(true, basedir + "code/pig/directory_test.pig", @@ -690,7 +732,8 @@ public class TestAvroStorage { verifyResults(createOutputName(),check); } - @Test public void testSeparatedByComma() throws Exception { + @Test + public void testSeparatedByComma() throws Exception { final String temp = basedir + "data/avro/uncompressed/testdirectory/part-m-0000"; StringBuffer sb = new StringBuffer(); @@ -709,7 +752,8 @@ public class TestAvroStorage { verifyResults(createOutputName(), check); } - @Test public void testDoubleUnderscore() throws Exception { + @Test + public void testDoubleUnderscore() throws Exception { final String input = basedir + "data/avro/uncompressed/records.avro"; final String check = basedir + "data/avro/uncompressed/recordsWithDoubleUnderscores.avro"; testAvroStorage(true, basedir + "code/pig/namesWithDoubleColons.pig", @@ -722,7 +766,8 @@ public class TestAvroStorage { verifyResults(createOutputName(),check); } - @Test public void testDoubleUnderscoreNoFlag() throws Exception { + @Test + public void testDoubleUnderscoreNoFlag() throws Exception { final String input = basedir + "data/avro/uncompressed/records.avro"; testAvroStorage(false, basedir + "code/pig/namesWithDoubleColons.pig", ImmutableMap.of( @@ -733,7 +778,8 @@ public class TestAvroStorage { ); } - @Test public void testLoadArrays() throws Exception { + @Test + public void testLoadArrays() throws Exception { final String input = basedir + "data/avro/uncompressed/arrays.avro"; final String check = basedir + "data/avro/uncompressed/arraysAsOutputByPig.avro"; testAvroStorage(true, basedir + "code/pig/identity_ao2.pig", @@ -746,7 +792,8 @@ public class TestAvroStorage { verifyResults(createOutputName(),check); } - @Test public void testLoadTrevniRecords() throws Exception { + @Test + public void testLoadTrevniRecords() throws Exception { final String input = basedir + "data/trevni/uncompressed/simpleRecordsTrevni.trevni"; final String check = basedir + "data/avro/uncompressed/simpleRecordsTrevni.avro"; testAvroStorage(true, basedir + "code/pig/trevni_to_avro.pig", @@ -758,7 +805,8 @@ public class TestAvroStorage { verifyResults(createOutputName(),check); } - @Test public void testLoadAndSaveTrevniRecords() throws Exception { + @Test + public void testLoadAndSaveTrevniRecords() throws Exception { final String input = basedir + "data/trevni/uncompressed/simpleRecordsTrevni.trevni"; final String check = basedir + "data/avro/uncompressed/simpleRecordsTrevni.avro"; @@ -933,7 +981,8 @@ public class TestAvroStorage { new Comparator<GenericData.Record>() { @Override public int compare(Record o1, Record o2) { - return o1.toString().compareTo(o2.toString()); + //return o1.compareTo(o2); throws AvroRuntimeException: Can't compare maps! + return o1.equals(o2) ? 0 : o1.toString().compareTo(o2.toString()); }} ); FileSystem fs = FileSystem.getLocal(new Configuration());
Modified: pig/branches/spark/test/org/apache/pig/builtin/TestOrcStorage.java URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/builtin/TestOrcStorage.java?rev=1654955&r1=1654954&r2=1654955&view=diff ============================================================================== --- pig/branches/spark/test/org/apache/pig/builtin/TestOrcStorage.java (original) +++ pig/branches/spark/test/org/apache/pig/builtin/TestOrcStorage.java Tue Jan 27 02:27:45 2015 @@ -22,6 +22,7 @@ import static org.junit.Assert.assertFal import static org.junit.Assert.assertTrue; import java.io.File; +import java.io.FileNotFoundException; import java.io.IOException; import java.math.BigDecimal; import java.util.ArrayList; @@ -31,8 +32,11 @@ import java.util.List; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.PathFilter; import org.apache.hadoop.hive.ql.io.orc.CompressionKind; import org.apache.hadoop.hive.ql.io.orc.OrcFile; import org.apache.hadoop.hive.ql.io.orc.OrcStruct; @@ -51,9 +55,7 @@ import org.apache.hadoop.io.FloatWritabl import org.apache.hadoop.io.IntWritable; import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.io.Text; -import org.apache.pig.ExecType; import org.apache.pig.PigServer; -import org.apache.pig.backend.executionengine.ExecException; import org.apache.pig.backend.hadoop.datastorage.ConfigurationUtil; import org.apache.pig.data.BinSedesTuple; import org.apache.pig.data.DataByteArray; @@ -96,8 +98,8 @@ public class TestOrcStorage { } @Before - public void setup() throws ExecException, IOException { - pigServer = new PigServer(ExecType.LOCAL); + public void setup() throws Exception { + pigServer = new PigServer(Util.getLocalTestMode()); fs = FileSystem.get(ConfigurationUtil.toConfiguration(pigServer.getPigContext().getProperties())); deleteTestFiles(); pigServer.mkdirs(outbasedir); @@ -189,8 +191,8 @@ public class TestOrcStorage { public void testSimpleStore() throws Exception { pigServer.registerQuery("A = load '" + INPUT1 + "' as (a0:int, a1:chararray);"); pigServer.store("A", OUTPUT1, "OrcStorage"); - Path outputFilePath = new Path(new Path(OUTPUT1), "part-m-00000"); - Reader reader = OrcFile.createReader(fs, outputFilePath); + + Reader reader = OrcFile.createReader(fs, Util.getFirstPartFile(new Path(OUTPUT1))); assertEquals(reader.getNumberOfRows(), 2); RecordReader rows = reader.rows(null); @@ -242,12 +244,12 @@ public class TestOrcStorage { pigServer.registerQuery("store B into '" + OUTPUT3 +"' using OrcStorage('-c SNAPPY');"); pigServer.executeBatch(); - Path outputFilePath = new Path(new Path(OUTPUT2), "part-r-00000"); + Path outputFilePath = Util.getFirstPartFile(new Path(OUTPUT2)); Reader reader = OrcFile.createReader(fs, outputFilePath); assertEquals(reader.getNumberOfRows(), 2); assertEquals(reader.getCompression(), CompressionKind.ZLIB); - Path outputFilePath2 = new Path(new Path(OUTPUT3), "part-r-00000"); + Path outputFilePath2 = Util.getFirstPartFile(new Path(OUTPUT3)); reader = OrcFile.createReader(fs, outputFilePath2); assertEquals(reader.getNumberOfRows(), 2); assertEquals(reader.getCompression(), CompressionKind.SNAPPY); Modified: pig/branches/spark/test/org/apache/pig/builtin/TestOrcStoragePushdown.java URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/builtin/TestOrcStoragePushdown.java?rev=1654955&r1=1654954&r2=1654955&view=diff ============================================================================== --- pig/branches/spark/test/org/apache/pig/builtin/TestOrcStoragePushdown.java (original) +++ pig/branches/spark/test/org/apache/pig/builtin/TestOrcStoragePushdown.java Tue Jan 27 02:27:45 2015 @@ -23,7 +23,6 @@ import static org.junit.Assert.assertTru import java.io.BufferedWriter; import java.io.File; import java.io.FileOutputStream; -import java.io.FileWriter; import java.io.IOException; import java.io.OutputStreamWriter; import java.math.BigDecimal; @@ -39,11 +38,9 @@ import org.apache.log4j.FileAppender; import org.apache.log4j.Level; import org.apache.log4j.Logger; import org.apache.log4j.SimpleLayout; -import org.apache.pig.ExecType; import org.apache.pig.Expression; import org.apache.pig.Expression.OpType; import org.apache.pig.PigServer; -import org.apache.pig.backend.executionengine.ExecException; import org.apache.pig.backend.executionengine.ExecJob; import org.apache.pig.impl.PigImplConstants; import org.apache.pig.impl.util.ObjectSerializer; @@ -122,7 +119,7 @@ public class TestOrcStoragePushdown { } private static void createInputData() throws Exception { - pigServer = new PigServer(ExecType.LOCAL); + pigServer = new PigServer(Util.getLocalTestMode()); new File(inpbasedir).mkdirs(); new File(outbasedir).mkdirs(); @@ -181,9 +178,9 @@ public class TestOrcStoragePushdown { } @Before - public void setup() throws ExecException{ + public void setup() throws Exception{ Util.resetStateForExecModeSwitch(); - pigServer = new PigServer(ExecType.LOCAL); + pigServer = new PigServer(Util.getLocalTestMode()); orcStorage = new OrcStorage(); } @@ -353,9 +350,9 @@ public class TestOrcStoragePushdown { } // For eclipse debugging - private void testPredicatePushdownLocal(String filterStmt, int expectedRows) throws IOException { + private void testPredicatePushdownLocal(String filterStmt, int expectedRows) throws Exception { - PigServer pigServer_disabledRule = new PigServer(ExecType.LOCAL); + PigServer pigServer_disabledRule = new PigServer(Util.getLocalTestMode()); // Test with PredicatePushdownOptimizer disabled. HashSet<String> disabledOptimizerRules = new HashSet<String>(); disabledOptimizerRules.add("PredicatePushdownOptimizer"); Modified: pig/branches/spark/test/org/apache/pig/builtin/TestPluckTuple.java URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/builtin/TestPluckTuple.java?rev=1654955&r1=1654954&r2=1654955&view=diff ============================================================================== --- pig/branches/spark/test/org/apache/pig/builtin/TestPluckTuple.java (original) +++ pig/branches/spark/test/org/apache/pig/builtin/TestPluckTuple.java Tue Jan 27 02:27:45 2015 @@ -26,12 +26,12 @@ import static org.junit.Assert.assertTru import java.util.Iterator; -import org.apache.pig.ExecType; import org.apache.pig.PigServer; import org.apache.pig.builtin.mock.Storage.Data; import org.apache.pig.data.Tuple; import org.apache.pig.impl.logicalLayer.schema.Schema; import org.apache.pig.impl.util.Utils; +import org.apache.pig.test.Util; import org.junit.Before; import org.junit.Test; @@ -40,7 +40,7 @@ public class TestPluckTuple { @Before public void setUp() throws Exception { - pigServer = new PigServer(ExecType.LOCAL); + pigServer = new PigServer(Util.getLocalTestMode()); } @Test Modified: pig/branches/spark/test/org/apache/pig/builtin/mock/TestMockStorage.java URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/builtin/mock/TestMockStorage.java?rev=1654955&r1=1654954&r2=1654955&view=diff ============================================================================== --- pig/branches/spark/test/org/apache/pig/builtin/mock/TestMockStorage.java (original) +++ pig/branches/spark/test/org/apache/pig/builtin/mock/TestMockStorage.java Tue Jan 27 02:27:45 2015 @@ -19,6 +19,7 @@ package org.apache.pig.builtin.mock; import static junit.framework.Assert.assertEquals; import static junit.framework.Assert.assertTrue; +import static junit.framework.Assert.assertFalse; import static junit.framework.Assert.fail; import static org.apache.pig.builtin.mock.Storage.resetData; import static org.apache.pig.builtin.mock.Storage.schema; @@ -28,19 +29,19 @@ import java.util.HashSet; import java.util.List; import java.util.Set; -import org.apache.pig.ExecType; import org.apache.pig.PigServer; import org.apache.pig.backend.executionengine.ExecJob; import org.apache.pig.backend.executionengine.ExecJob.JOB_STATUS; import org.apache.pig.builtin.mock.Storage.Data; import org.apache.pig.data.Tuple; +import org.apache.pig.test.Util; import org.junit.Test; public class TestMockStorage { @Test public void testMockStoreAndLoad() throws Exception { - PigServer pigServer = new PigServer(ExecType.LOCAL); + PigServer pigServer = new PigServer(Util.getLocalTestMode()); Data data = resetData(pigServer); data.set("foo", @@ -60,7 +61,7 @@ public class TestMockStorage { @Test public void testMockSchema() throws Exception { - PigServer pigServer = new PigServer(ExecType.LOCAL); + PigServer pigServer = new PigServer(Util.getLocalTestMode()); Data data = resetData(pigServer); data.set("foo", "blah:chararray", @@ -83,7 +84,7 @@ public class TestMockStorage { @Test public void testMockStoreUnion() throws Exception { - PigServer pigServer = new PigServer(ExecType.LOCAL); + PigServer pigServer = new PigServer(Util.getLocalTestMode()); Data data = resetData(pigServer); data.set("input1", @@ -122,7 +123,7 @@ public class TestMockStorage { @Test public void testBadUsage1() throws Exception { - PigServer pigServer = new PigServer(ExecType.LOCAL); + PigServer pigServer = new PigServer(Util.getLocalTestMode()); Data data = resetData(pigServer); data.set("input1", @@ -145,7 +146,7 @@ public class TestMockStorage { @Test public void testBadUsage2() throws Exception { - PigServer pigServer = new PigServer(ExecType.LOCAL); + PigServer pigServer = new PigServer(Util.getLocalTestMode()); Data data = resetData(pigServer); data.set("input", @@ -168,6 +169,10 @@ public class TestMockStorage { break; } } + if (Util.getLocalTestMode().toString().equals("TEZ_LOCAL")) { + assertFalse("job should have success in tez_local mode since we have two vertex, each for a single store", failed); + return; + } assertTrue("job should have failed for storing twice in the same location", failed); } Modified: pig/branches/spark/test/org/apache/pig/impl/builtin/TestStreamingUDF.java URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/impl/builtin/TestStreamingUDF.java?rev=1654955&r1=1654954&r2=1654955&view=diff ============================================================================== --- pig/branches/spark/test/org/apache/pig/impl/builtin/TestStreamingUDF.java (original) +++ pig/branches/spark/test/org/apache/pig/impl/builtin/TestStreamingUDF.java Tue Jan 27 02:27:45 2015 @@ -26,7 +26,6 @@ import java.math.BigInteger; import java.util.Iterator; import java.util.List; -import org.apache.pig.ExecType; import org.apache.pig.PigServer; import org.apache.pig.builtin.mock.Storage.Data; import org.apache.pig.data.DataBag; @@ -34,12 +33,26 @@ import org.apache.pig.data.Tuple; import org.apache.pig.data.TupleFactory; import org.apache.pig.test.MiniGenericCluster; import org.apache.pig.test.Util; +import org.apache.pig.test.junit.OrderedJUnit4Runner; +import org.apache.pig.test.junit.OrderedJUnit4Runner.TestOrder; import org.joda.time.DateTime; import org.junit.AfterClass; import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; +import org.junit.runner.RunWith; +// Need to run testPythonUDF_onCluster first due to TEZ-1802 +@RunWith(OrderedJUnit4Runner.class) +@TestOrder({ + "testPythonUDF_onCluster", + "testPythonUDF__allTypes", + "testPythonUDF__withBigDecimal", + "testPythonUDF", + "testPythonUDF__withBigInteger", + "testPythonUDF__withDateTime", + "testPythonUDF_withNewline" +}) public class TestStreamingUDF { private static PigServer pigServerLocal = null; private static PigServer pigServerMapReduce = null; @@ -101,7 +114,7 @@ public class TestStreamingUDF { @Test public void testPythonUDF() throws Exception { - pigServerLocal = new PigServer(ExecType.LOCAL); + pigServerLocal = new PigServer(Util.getLocalTestMode()); String[] pythonScript = { "from pig_util import outputSchema", @@ -136,7 +149,7 @@ public class TestStreamingUDF { @Test public void testPythonUDF_withNewline() throws Exception { - pigServerLocal = new PigServer(ExecType.LOCAL); + pigServerLocal = new PigServer(Util.getLocalTestMode()); String[] pythonScript = { "from pig_util import outputSchema", @@ -171,7 +184,7 @@ public class TestStreamingUDF { @Test public void testPythonUDF__withBigInteger() throws Exception { - pigServerLocal = new PigServer(ExecType.LOCAL); + pigServerLocal = new PigServer(Util.getLocalTestMode()); String[] pythonScript = { "from pig_util import outputSchema", @@ -199,7 +212,7 @@ public class TestStreamingUDF { @Test public void testPythonUDF__withBigDecimal() throws Exception { - pigServerLocal = new PigServer(ExecType.LOCAL); + pigServerLocal = new PigServer(Util.getLocalTestMode()); String[] pythonScript = { "from pig_util import outputSchema", @@ -229,7 +242,7 @@ public class TestStreamingUDF { @Test public void testPythonUDF__withDateTime() throws Exception { - pigServerLocal = new PigServer(ExecType.LOCAL); + pigServerLocal = new PigServer(Util.getLocalTestMode()); String[] pythonScript = { "from pig_util import outputSchema", @@ -257,7 +270,7 @@ public class TestStreamingUDF { @Test public void testPythonUDF__allTypes() throws Exception { - pigServerLocal = new PigServer(ExecType.LOCAL); + pigServerLocal = new PigServer(Util.getLocalTestMode()); String[] pythonScript = { "# -*- coding: utf-8 -*-", Modified: pig/branches/spark/test/org/apache/pig/newplan/logical/optimizer/TestImplicitSplitOnTuple.java URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/newplan/logical/optimizer/TestImplicitSplitOnTuple.java?rev=1654955&r1=1654954&r2=1654955&view=diff ============================================================================== --- pig/branches/spark/test/org/apache/pig/newplan/logical/optimizer/TestImplicitSplitOnTuple.java (original) +++ pig/branches/spark/test/org/apache/pig/newplan/logical/optimizer/TestImplicitSplitOnTuple.java Tue Jan 27 02:27:45 2015 @@ -18,16 +18,11 @@ package org.apache.pig.newplan.logical.optimizer; -import static org.apache.pig.ExecType.LOCAL; import static org.apache.pig.builtin.mock.Storage.tuple; import static org.junit.Assert.assertEquals; -import java.io.File; -import java.io.IOException; -import java.util.Collections; import java.util.List; -import org.apache.pig.PigRunner; import org.apache.pig.PigServer; import org.apache.pig.builtin.mock.Storage; import org.apache.pig.builtin.mock.Storage.Data; @@ -39,8 +34,8 @@ import org.junit.Test; public class TestImplicitSplitOnTuple { @Test - public void testImplicitSplitterOnTuple() throws IOException { - PigServer pigServer = new PigServer(LOCAL); + public void testImplicitSplitterOnTuple() throws Exception { + PigServer pigServer = new PigServer(Util.getLocalTestMode()); Data data = Storage.resetData(pigServer); data.set("input", tuple("1", "1001", "101"), Modified: pig/branches/spark/test/org/apache/pig/newplan/logical/optimizer/TestSchemaResetter.java URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/newplan/logical/optimizer/TestSchemaResetter.java?rev=1654955&r1=1654954&r2=1654955&view=diff ============================================================================== --- pig/branches/spark/test/org/apache/pig/newplan/logical/optimizer/TestSchemaResetter.java (original) +++ pig/branches/spark/test/org/apache/pig/newplan/logical/optimizer/TestSchemaResetter.java Tue Jan 27 02:27:45 2015 @@ -18,7 +18,6 @@ package org.apache.pig.newplan.logical.optimizer; -import static org.apache.pig.ExecType.LOCAL; import static org.apache.pig.builtin.mock.Storage.tuple; import static org.junit.Assert.assertEquals; @@ -68,8 +67,8 @@ public class TestSchemaResetter { } @Test - public void testSchemaResetterExec() throws IOException { - PigServer pigServer = new PigServer(LOCAL); + public void testSchemaResetterExec() throws Exception { + PigServer pigServer = new PigServer(Util.getLocalTestMode()); Data data = Storage.resetData(pigServer); data.set("input", tuple(tuple("1", "2")), Modified: pig/branches/spark/test/org/apache/pig/newplan/logical/relational/TestLocationInPhysicalPlan.java URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/newplan/logical/relational/TestLocationInPhysicalPlan.java?rev=1654955&r1=1654954&r2=1654955&view=diff ============================================================================== --- pig/branches/spark/test/org/apache/pig/newplan/logical/relational/TestLocationInPhysicalPlan.java (original) +++ pig/branches/spark/test/org/apache/pig/newplan/logical/relational/TestLocationInPhysicalPlan.java Tue Jan 27 02:27:45 2015 @@ -23,7 +23,6 @@ import java.util.List; import junit.framework.Assert; -import org.apache.pig.ExecType; import org.apache.pig.PigServer; import org.apache.pig.backend.executionengine.ExecJob; import org.apache.pig.backend.hadoop.executionengine.physicalLayer.PhysicalOperator.OriginalLocation; @@ -46,7 +45,7 @@ public class TestLocationInPhysicalPlan "3,1,1", "1,2,1", }); - PigServer pigServer = new PigServer(ExecType.LOCAL); + PigServer pigServer = new PigServer(Util.getLocalTestMode()); pigServer.setBatchOn(); pigServer.registerQuery( "A = LOAD '" + Util.encodeEscape(input.getAbsolutePath()) + "' using PigStorage();\n" @@ -61,6 +60,10 @@ public class TestLocationInPhysicalPlan Assert.assertEquals(0, originalLocation.getOffset()); Assert.assertEquals("A", originalLocation.getAlias()); JobStats jStats = (JobStats)job.getStatistics().getJobGraph().getSinks().get(0); - Assert.assertEquals("M: A[1,4],A[3,4],B[2,4] C: A[3,4],B[2,4] R: A[3,4]", jStats.getAliasLocation()); + if (Util.getLocalTestMode().toString().equals("TEZ_LOCAL")) { + Assert.assertEquals("A[1,4],A[3,4],B[2,4]", jStats.getAliasLocation()); + } else { + Assert.assertEquals("M: A[1,4],A[3,4],B[2,4] C: A[3,4],B[2,4] R: A[3,4]", jStats.getAliasLocation()); + } } } Modified: pig/branches/spark/test/org/apache/pig/parser/TestColumnAliasConversion.java URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/parser/TestColumnAliasConversion.java?rev=1654955&r1=1654954&r2=1654955&view=diff ============================================================================== --- pig/branches/spark/test/org/apache/pig/parser/TestColumnAliasConversion.java (original) +++ pig/branches/spark/test/org/apache/pig/parser/TestColumnAliasConversion.java Tue Jan 27 02:27:45 2015 @@ -159,6 +159,22 @@ public class TestColumnAliasConversion { Assert.fail( "Query should fail to validate." ); } + @Test + public void testInvalidNestedProjection() throws Exception { + String query = "A = load 'x' as (field);" + + "B = foreach A {" + + " C = LIMIT invalidName 1;" + + " generate C.foo;" + + "};"; + try { + validate( query ); + } catch(PlanValidationException ex) { + System.out.println(ex.getMessage()); + return; + } + Assert.fail( "Query should fail to validate." ); + } + private LogicalPlan validate(String query) throws RecognitionException, ParsingFailureException, IOException { LogicalPlan plan = ParserTestingUtils.generateLogicalPlan( query ); ColumnAliasConversionVisitor visitor = new ColumnAliasConversionVisitor( plan ); Modified: pig/branches/spark/test/org/apache/pig/parser/TestLogicalPlanGenerator.java URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/parser/TestLogicalPlanGenerator.java?rev=1654955&r1=1654954&r2=1654955&view=diff ============================================================================== --- pig/branches/spark/test/org/apache/pig/parser/TestLogicalPlanGenerator.java (original) +++ pig/branches/spark/test/org/apache/pig/parser/TestLogicalPlanGenerator.java Tue Jan 27 02:27:45 2015 @@ -29,7 +29,6 @@ import java.util.Iterator; import java.util.List; import org.antlr.runtime.MismatchedTokenException; -import org.apache.pig.ExecType; import org.apache.pig.PigServer; import org.apache.pig.builtin.mock.Storage.Data; import org.apache.pig.data.Tuple; @@ -50,7 +49,7 @@ public class TestLogicalPlanGenerator { @Before public void setUp() throws Exception { - pigServer = new PigServer(ExecType.LOCAL); + pigServer = new PigServer(Util.getLocalTestMode()); } @BeforeClass Modified: pig/branches/spark/test/org/apache/pig/pigunit/PigTest.java URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/pigunit/PigTest.java?rev=1654955&r1=1654954&r2=1654955&view=diff ============================================================================== --- pig/branches/spark/test/org/apache/pig/pigunit/PigTest.java (original) +++ pig/branches/spark/test/org/apache/pig/pigunit/PigTest.java Tue Jan 27 02:27:45 2015 @@ -18,14 +18,20 @@ import java.io.FileInputStream; import java.io.IOException; import java.io.PrintWriter; import java.io.StringReader; +import java.lang.Object; import java.nio.MappedByteBuffer; import java.nio.channels.FileChannel; import java.nio.charset.Charset; +import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; import java.util.Iterator; +import java.util.List; import java.util.Map; +import java.util.Set; +import com.google.common.collect.Iterables; import junit.framework.Assert; import org.apache.commons.lang.StringUtils; @@ -38,6 +44,7 @@ import org.apache.pig.data.DataType; import org.apache.pig.data.Tuple; import org.apache.pig.impl.PigContext; import org.apache.pig.impl.io.FileLocalizer; +import org.apache.pig.impl.logicalLayer.FrontendException; import org.apache.pig.impl.logicalLayer.schema.Schema; import org.apache.pig.pigunit.pig.PigServer; import org.apache.pig.tools.parameters.ParseException; @@ -254,37 +261,167 @@ public class PigTest { aliasOverrides.remove(alias); } - public void assertOutput(String[] expected) throws IOException, ParseException { + /** + * Returns a Map that has alias to it's schema. + * + * @return A map that has alias name as a key and the alias's schema as a value + * @throws FrontendException If there was an error dumping the schema + */ + public Map<String, String> getAliasToSchemaMap() throws FrontendException, IOException, ParseException { + HashMap<String, String> aliasSchemas = new HashMap<String, String>(); registerScript(); - String alias = aliasOverrides.get("LAST_STORE_ALIAS"); + PigServer server = getPigServer(); + Set<String> aliasKeySet = server.getAliasKeySet(); + for (String alias: aliasKeySet) { + try { + StringBuilder tsb = new StringBuilder(); + Schema.stringifySchema(tsb, server.dumpSchema(alias), DataType.TUPLE, Integer.MIN_VALUE); + aliasSchemas.put(alias, tsb.toString()); + } catch (FrontendException e) { + /** + * If PigServer fails to describe a schema for an alias a FrontendException is thrown. + * PigServer.getAliasKeySet() returns aliases that cannot have their schema described. + * We want to skip over these particular aliases. + */ + if (e.getErrorCode() == 1001) { + //Let's print a warning + System.out.println(e.getMessage()); + } else { + throw e; + } + } + } + return aliasSchemas; + } - assertEquals(StringUtils.join(expected, "\n"), StringUtils.join(getAliasFromCache(alias), "\n")); + /** + * Creates a temp file and populates it with the specified mock data. + * + * @param alias alias that the temp file is for + * @param mockData data that is being mocked for the alias + * @return path to the temp file + */ + private String makeMockTempFile(String alias, String[] mockData) throws IOException { + //The FileLocalizer uses a random variable, but that can have collisions. By using the current time, thread, + //and alias we should be able to guaratee a unique file + String uniqueSuffix = alias + "." + System.currentTimeMillis() + "." + Thread.currentThread().getId(); + //PigServer/Cluster is not initialized yet. Let's initialize it. + if (getPigServer() == null) { + getCluster(); + } + String path = FileLocalizer.getTemporaryPath(getPigServer().getPigContext(), uniqueSuffix).toString(); + getCluster().copyFromLocalFile(mockData, path, true); + return path; } - public void assertOutput(String alias, String[] expected) throws IOException, ParseException { - registerScript(); + private String getActualResults(String alias, boolean ignoreOrder) throws IOException, ParseException { + //Tuples are sortable, but we should sort it as Strings for convenience when comparing to expected data + Iterator<Tuple> iterator = getAliasFromCache(alias); + List<String> actualResults = new ArrayList<String>(); + while (iterator.hasNext()) { + actualResults.add(iterator.next().toString()); + } + + if (ignoreOrder) { + Collections.sort(actualResults); + } + return StringUtils.join(actualResults, "\n"); + } - assertEquals(StringUtils.join(expected, "\n"), StringUtils.join(getAliasFromCache(alias), "\n")); + /** + * Allows you to mock a specific alias. + * + * This method will create a temporary file on the system that contains the mock data. It will then change the pig + * script to actually replace the alias when being assigned a value with a load file command which loads the temporary + * file. + * + * @param alias The alias to be mocked + * @param mockData The data you wished to be contained in the alias where each element in the array is a tab-delimited + * @param aliasSchema The schema of the alias provided. Your mockData should fit this schema + */ + public void mockAlias(String alias, String[] mockData, String aliasSchema) throws IOException { + mockAlias(alias, mockData, aliasSchema, "\\t"); } - public void assertOutput(File expected) throws IOException, ParseException { + /** + * Allows you to mock a specific alias. + * + * This method will create a temporary file on the system that contains the mock data. It will then change the pig + * script to actually replace the alias when being assigned a value with a load file command which loads the temporary + * file. + * + * @param alias The alias to be mocked + * @param mockData String array where each element is an entry in the alias and the line has its elements delimited + * by the value provided by delimiter. + * @param aliasSchema This is the schema of the alias being mocked + * @param delimiter The delimiter used to separate data in mockData + */ + public void mockAlias(String alias, String[] mockData, String aliasSchema, String delimiter) throws IOException { + String mockFile = makeMockTempFile(alias, mockData); + override(alias, String.format("%s = LOAD '%s' USING PigStorage('%s') AS %s;", alias, mockFile, delimiter, aliasSchema)); + } + + /** + * Compares the expected results to the results of the last alias generated in the script. Order does not matter + * and as long as the result is located in any index of expected and any line of the output then this will pass. + * + * @param expected The expected results + */ + public void assertOutputAnyOrder(String[] expected) throws IOException, ParseException { + assertOutput(expected, true); + } + + public void assertOutput(String[] expected) throws IOException, ParseException { + assertOutput(expected, false); + } + + private void assertOutput(String[] expected, boolean ignoreOrder) throws IOException, ParseException { registerScript(); String alias = aliasOverrides.get("LAST_STORE_ALIAS"); - assertEquals(readFile(expected).replaceAll("\r\n", "\n"), StringUtils.join(getAliasFromCache(alias), "\n")); + if (ignoreOrder) { + Arrays.sort(expected); + } + assertEquals(StringUtils.join(expected, "\n"), getActualResults(alias, ignoreOrder)); + } + + /** + * Compares the expected results to the results of the provided alias's output. Order does not matter + * and as long as the result is located in any index of expected and any line of the output then this will pass. + * + * @param alias The alias whose results we want to check + * @param expected The expected results + */ + public void assertOutputAnyOrder(String alias, String[] expected) throws IOException, ParseException { + assertOutput(alias, expected, true); + } + + public void assertOutput(String alias, String[] expected) throws IOException, ParseException { + assertOutput(alias, expected, false); } - public void assertOutput(String alias, File expected) throws IOException, ParseException { + private void assertOutput(String alias, String[] expected, boolean ignoreOrder) throws IOException, ParseException { registerScript(); - assertEquals(readFile(expected).replaceAll("\r\n", "\n"), StringUtils.join(getAliasFromCache(alias), "\n")); + if (ignoreOrder) { + Arrays.sort(expected); + } + assertEquals(StringUtils.join(expected, "\n"), getActualResults(alias, ignoreOrder)); } + public void assertOutput(File expected) throws IOException, ParseException { + assertOutput(readFile(expected).split("(\\r\\n|\\n)"), false); + } + + public void assertOutput(String alias, File expected) throws IOException, ParseException { + assertOutput(alias, readFile(expected).split("(\\r\\n|\\n)"), false); + } + public void assertOutput(String aliasInput, String[] input, String alias, String[] expected) throws IOException, ParseException { assertOutput(aliasInput, input, alias, expected, "\\t"); } - + public void assertOutput(String aliasInput, String[] input, String alias, String[] expected, String delimiter) throws IOException, ParseException { registerScript(); @@ -292,18 +429,15 @@ public class PigTest { StringBuilder sb = new StringBuilder(); Schema.stringifySchema(sb, getPigServer().dumpSchema(aliasInput), DataType.TUPLE) ; - final String destination = FileLocalizer.getTemporaryPath(getPigServer().getPigContext()).toString(); - getCluster().copyFromLocalFile(input, destination, true); - override(aliasInput, - String.format("%s = LOAD '%s' USING PigStorage('%s') AS %s;", aliasInput, destination, delimiter, sb.toString())); + mockAlias(aliasInput, input, sb.toString(), delimiter); - assertOutput(alias, expected); + assertOutput(alias, expected, false); } protected void assertEquals(String expected, String current) { Assert.assertEquals(expected, current); } - + private static String readFile(String path) throws IOException { return readFile(new File(path)); } Modified: pig/branches/spark/test/org/apache/pig/test/MiniGenericCluster.java URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/test/MiniGenericCluster.java?rev=1654955&r1=1654954&r2=1654955&view=diff ============================================================================== --- pig/branches/spark/test/org/apache/pig/test/MiniGenericCluster.java (original) +++ pig/branches/spark/test/org/apache/pig/test/MiniGenericCluster.java Tue Jan 27 02:27:45 2015 @@ -25,6 +25,7 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.pig.ExecType; import org.apache.pig.backend.hadoop.datastorage.ConfigurationUtil; +import org.apache.pig.backend.hadoop.executionengine.Launcher; /** * This class builds a single instance of itself with the Singleton @@ -149,4 +150,18 @@ abstract public class MiniGenericCluster String msg = "function called on MiniCluster that has been shutdown"; throw new RuntimeException(msg); } + + static public Launcher getLauncher() { + String execType = System.getProperty("test.exec.type"); + if (execType == null) { + System.setProperty("test.exec.type", EXECTYPE_MR); + } + if (execType.equalsIgnoreCase(EXECTYPE_MR)) { + return MiniCluster.getLauncher(); + } else if (execType.equalsIgnoreCase(EXECTYPE_TEZ)) { + return TezMiniCluster.getLauncher(); + } else { + throw new RuntimeException("Unknown test.exec.type: " + execType); + } + } } Modified: pig/branches/spark/test/org/apache/pig/test/TestAlgebraicEvalLocal.java URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/test/TestAlgebraicEvalLocal.java?rev=1654955&r1=1654954&r2=1654955&view=diff ============================================================================== --- pig/branches/spark/test/org/apache/pig/test/TestAlgebraicEvalLocal.java (original) +++ pig/branches/spark/test/org/apache/pig/test/TestAlgebraicEvalLocal.java Tue Jan 27 02:27:45 2015 @@ -26,7 +26,6 @@ import java.util.Iterator; import java.util.Properties; import java.util.Random; -import org.apache.pig.ExecType; import org.apache.pig.PigServer; import org.apache.pig.builtin.PigStorage; import org.apache.pig.data.DataType; @@ -42,7 +41,7 @@ public class TestAlgebraicEvalLocal { @Before public void setUp() throws Exception { - pig = new PigServer(ExecType.LOCAL, new Properties()); + pig = new PigServer(Util.getLocalTestMode(), new Properties()); } Boolean[] nullFlags = new Boolean[]{ false, true}; Modified: pig/branches/spark/test/org/apache/pig/test/TestAssert.java URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/test/TestAssert.java?rev=1654955&r1=1654954&r2=1654955&view=diff ============================================================================== --- pig/branches/spark/test/org/apache/pig/test/TestAssert.java (original) +++ pig/branches/spark/test/org/apache/pig/test/TestAssert.java Tue Jan 27 02:27:45 2015 @@ -24,12 +24,9 @@ import static org.apache.pig.builtin.moc import java.io.ByteArrayInputStream; import java.io.InputStream; import java.util.List; -import java.util.Properties; import junit.framework.Assert; -import org.apache.pig.ExecType; -import org.apache.pig.PigConfiguration; import org.apache.pig.PigServer; import org.apache.pig.builtin.mock.Storage.Data; import org.apache.pig.data.Tuple; @@ -44,7 +41,7 @@ public class TestAssert { */ @Test public void testPositive() throws Exception { - PigServer pigServer = new PigServer(ExecType.LOCAL); + PigServer pigServer = new PigServer(Util.getLocalTestMode()); Data data = resetData(pigServer); data.set("foo", @@ -74,7 +71,7 @@ public class TestAssert { */ @Test public void testInScript() throws Exception { - PigServer pigServer = new PigServer(ExecType.LOCAL); + PigServer pigServer = new PigServer(Util.getLocalTestMode()); Data data = resetData(pigServer); data.set("foo", @@ -104,7 +101,7 @@ public class TestAssert { */ @Test public void testNegative() throws Exception { - PigServer pigServer = new PigServer(ExecType.LOCAL); + PigServer pigServer = new PigServer(Util.getLocalTestMode()); Data data = resetData(pigServer); data.set("foo", @@ -130,7 +127,7 @@ public class TestAssert { */ @Test public void testNegativeWithoutFetch() throws Exception { - PigServer pigServer = new PigServer(ExecType.LOCAL); + PigServer pigServer = new PigServer(Util.getLocalTestMode()); Data data = resetData(pigServer); data.set("foo", @@ -156,7 +153,7 @@ public class TestAssert { */ @Test(expected=FrontendException.class) public void testNegativeWithAlias() throws Exception { - PigServer pigServer = new PigServer(ExecType.LOCAL); + PigServer pigServer = new PigServer(Util.getLocalTestMode()); Data data = resetData(pigServer); data.set("foo", Modified: pig/branches/spark/test/org/apache/pig/test/TestAutoLocalMode.java URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/test/TestAutoLocalMode.java?rev=1654955&r1=1654954&r2=1654955&view=diff ============================================================================== --- pig/branches/spark/test/org/apache/pig/test/TestAutoLocalMode.java (original) +++ pig/branches/spark/test/org/apache/pig/test/TestAutoLocalMode.java Tue Jan 27 02:27:45 2015 @@ -33,7 +33,6 @@ import org.apache.log4j.FileAppender; import org.apache.log4j.Level; import org.apache.log4j.Logger; import org.apache.log4j.SimpleLayout; -import org.apache.pig.ExecType; import org.apache.pig.PigConfiguration; import org.apache.pig.PigServer; import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.JobControlCompiler; @@ -94,7 +93,7 @@ public class TestAutoLocalMode { @Before public void setUp() throws Exception{ - pigServer = new PigServer(ExecType.MAPREDUCE, cluster.getProperties()); + pigServer = new PigServer(cluster.getExecType(), cluster.getProperties()); pigServer.getPigContext().getExecutionEngine().setProperty(PigConfiguration.PIG_OPT_FETCH, "false"); pigServer.getPigContext().getExecutionEngine().setProperty(PigConfiguration.PIG_AUTO_LOCAL_ENABLED, String.valueOf("true")); pigServer.getPigContext().getExecutionEngine().setProperty(PigConfiguration.PIG_AUTO_LOCAL_INPUT_MAXBYTES, "200"); Modified: pig/branches/spark/test/org/apache/pig/test/TestBatchAliases.java URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/test/TestBatchAliases.java?rev=1654955&r1=1654954&r2=1654955&view=diff ============================================================================== --- pig/branches/spark/test/org/apache/pig/test/TestBatchAliases.java (original) +++ pig/branches/spark/test/org/apache/pig/test/TestBatchAliases.java Tue Jan 27 02:27:45 2015 @@ -23,7 +23,6 @@ import static org.junit.Assert.assertTru import java.io.IOException; import java.util.Properties; -import org.apache.pig.ExecType; import org.apache.pig.PigConfiguration; import org.apache.pig.PigServer; import org.apache.pig.backend.executionengine.ExecJob; @@ -39,7 +38,7 @@ public class TestBatchAliases { @Before public void setUp() throws Exception { System.setProperty(PigConfiguration.PIG_OPT_MULTIQUERY, ""+true); - myPig = new PigServer(ExecType.LOCAL, new Properties()); + myPig = new PigServer(Util.getLocalTestMode(), new Properties()); deleteOutputFiles(); } Modified: pig/branches/spark/test/org/apache/pig/test/TestBlackAndWhitelistValidator.java URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/test/TestBlackAndWhitelistValidator.java?rev=1654955&r1=1654954&r2=1654955&view=diff ============================================================================== --- pig/branches/spark/test/org/apache/pig/test/TestBlackAndWhitelistValidator.java (original) +++ pig/branches/spark/test/org/apache/pig/test/TestBlackAndWhitelistValidator.java Tue Jan 27 02:27:45 2015 @@ -26,7 +26,6 @@ import java.io.File; import java.util.Properties; import org.apache.commons.io.IOUtils; -import org.apache.pig.ExecType; import org.apache.pig.FuncSpec; import org.apache.pig.PigConfiguration; import org.apache.pig.PigServer; @@ -61,7 +60,7 @@ public class TestBlackAndWhitelistValida @Before public void setUp() throws Exception { - ctx = new PigContext(ExecType.LOCAL, new Properties()); + ctx = new PigContext(Util.getLocalTestMode(), new Properties()); ctx.connect(); } Modified: pig/branches/spark/test/org/apache/pig/test/TestBuiltInBagToTupleOrString.java URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/test/TestBuiltInBagToTupleOrString.java?rev=1654955&r1=1654954&r2=1654955&view=diff ============================================================================== --- pig/branches/spark/test/org/apache/pig/test/TestBuiltInBagToTupleOrString.java (original) +++ pig/branches/spark/test/org/apache/pig/test/TestBuiltInBagToTupleOrString.java Tue Jan 27 02:27:45 2015 @@ -22,7 +22,6 @@ import static junit.framework.Assert.*; import java.util.Iterator; import java.util.List; -import org.apache.pig.ExecType; import org.apache.pig.PigServer; import org.apache.pig.backend.executionengine.ExecException; import org.apache.pig.builtin.BagToString; @@ -407,7 +406,7 @@ public class TestBuiltInBagToTupleOrStri @Test public void testPigScriptForBagToTupleUDF() throws Exception { - PigServer pigServer = new PigServer(ExecType.LOCAL); + PigServer pigServer = new PigServer(Util.getLocalTestMode()); Data data = resetData(pigServer); // bag of chararray @@ -436,7 +435,7 @@ public class TestBuiltInBagToTupleOrStri @Test public void testPigScriptMultipleElmementsPerTupleForBagTupleUDF() throws Exception { - PigServer pigServer = new PigServer(ExecType.LOCAL); + PigServer pigServer = new PigServer(Util.getLocalTestMode()); Data data = resetData(pigServer); data.set("foo", "myBag:bag{t:(l:chararray)}", @@ -451,7 +450,7 @@ public class TestBuiltInBagToTupleOrStri @Test public void testPigScriptNestedTupleForBagToTupleDF() throws Exception { - PigServer pigServer = new PigServer(ExecType.LOCAL); + PigServer pigServer = new PigServer(Util.getLocalTestMode()); Data data = resetData(pigServer); Tuple nestedTuple = tuple(bag(tuple("c"), tuple("d"))); @@ -469,7 +468,7 @@ public class TestBuiltInBagToTupleOrStri @Test public void testPigScriptEmptyBagForBagToTupleUDF() throws Exception { - PigServer pigServer = new PigServer(ExecType.LOCAL); + PigServer pigServer = new PigServer(Util.getLocalTestMode()); Data data = resetData(pigServer); data.set("foo", "myBag:bag{t:(l:chararray)}", @@ -487,7 +486,7 @@ public class TestBuiltInBagToTupleOrStri @Test public void testPigScriptrForBagToStringUDF() throws Exception { - PigServer pigServer = new PigServer(ExecType.LOCAL); + PigServer pigServer = new PigServer(Util.getLocalTestMode()); Data data = resetData(pigServer); data.set("foo", "myBag:bag{t:(l:chararray)}", @@ -509,7 +508,7 @@ public class TestBuiltInBagToTupleOrStri @Test public void testPigScriptMultipleElmementsPerTupleForBagToStringUDF() throws Exception { - PigServer pigServer = new PigServer(ExecType.LOCAL); + PigServer pigServer = new PigServer(Util.getLocalTestMode()); Data data = resetData(pigServer); data.set("foo", "myBag:bag{t:(l:chararray)}", @@ -530,7 +529,7 @@ public class TestBuiltInBagToTupleOrStri @Test public void testPigScriptNestedTupleForBagToStringUDF() throws Exception { - PigServer pigServer = new PigServer(ExecType.LOCAL); + PigServer pigServer = new PigServer(Util.getLocalTestMode()); Data data = resetData(pigServer); Tuple nestedTuple = tuple(bag(tuple("c"), tuple("d"))); @@ -548,7 +547,7 @@ public class TestBuiltInBagToTupleOrStri @Test public void testPigScriptEmptyBagForBagToStringUDF() throws Exception { - PigServer pigServer = new PigServer(ExecType.LOCAL); + PigServer pigServer = new PigServer(Util.getLocalTestMode()); Data data = resetData(pigServer); data.set("foo", "myBag:bag{t:(l:chararray)}", Modified: pig/branches/spark/test/org/apache/pig/test/TestBuiltin.java URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/test/TestBuiltin.java?rev=1654955&r1=1654954&r2=1654955&view=diff ============================================================================== --- pig/branches/spark/test/org/apache/pig/test/TestBuiltin.java (original) +++ pig/branches/spark/test/org/apache/pig/test/TestBuiltin.java Tue Jan 27 02:27:45 2015 @@ -42,7 +42,6 @@ import java.math.BigInteger; import org.apache.pig.Algebraic; import org.apache.pig.EvalFunc; -import org.apache.pig.ExecType; import org.apache.pig.LoadFunc; import org.apache.pig.PigServer; import org.apache.pig.backend.executionengine.ExecException; @@ -204,7 +203,7 @@ public class TestBuiltin { public void setUp() throws Exception { Util.resetStateForExecModeSwitch(); - pigServer = new PigServer(ExecType.LOCAL, new Properties()); + pigServer = new PigServer(Util.getLocalTestMode(), new Properties()); pigServer.setValidateEachStatement(true); // First set up data structs for "base" SUM, MIN and MAX and AVG. // The allowed input and expected output data structs for @@ -2411,7 +2410,7 @@ public class TestBuiltin { public void testComplexMultiCONCAT() throws Exception { String input = "vararg_concat_test_jira_3444.txt"; Util.createLocalInputFile(input, new String[]{"dummy"}); - PigServer pigServer = new PigServer(ExecType.LOCAL); + PigServer pigServer = new PigServer(Util.getLocalTestMode()); pigServer.registerQuery("A = LOAD '"+input+"' as (x:chararray);"); pigServer.registerQuery("B = foreach A generate CONCAT('a', CONCAT('b',CONCAT('c','d')));"); @@ -2483,7 +2482,7 @@ public class TestBuiltin { // String input = "vararg_sprintf_test_jira_3939.txt"; Util.createLocalInputFile(input, new String[]{"dummy"}); - PigServer pigServer = new PigServer(ExecType.LOCAL); + PigServer pigServer = new PigServer(Util.getLocalTestMode()); pigServer.registerQuery("A = LOAD '"+input+"' as (x:chararray);"); // pigServer.registerQuery("B = foreach A generate SPRINTF('%6s|%-8s|%2$,+12d %2$8x', 'yay', 665568);"); Modified: pig/branches/spark/test/org/apache/pig/test/TestBuiltinInvoker.java URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/test/TestBuiltinInvoker.java?rev=1654955&r1=1654954&r2=1654955&view=diff ============================================================================== --- pig/branches/spark/test/org/apache/pig/test/TestBuiltinInvoker.java (original) +++ pig/branches/spark/test/org/apache/pig/test/TestBuiltinInvoker.java Tue Jan 27 02:27:45 2015 @@ -27,7 +27,6 @@ import java.util.List; import java.util.Random; import java.util.Set; -import org.apache.pig.ExecType; import org.apache.pig.PigServer; import org.apache.pig.builtin.mock.Storage.Data; import org.apache.pig.data.Tuple; @@ -50,7 +49,7 @@ public class TestBuiltinInvoker { @Before public void setUp() throws Exception { - pigServer = new PigServer(ExecType.LOCAL); + pigServer = new PigServer(Util.getLocalTestMode()); data = resetData(pigServer); Modified: pig/branches/spark/test/org/apache/pig/test/TestCase.java URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/test/TestCase.java?rev=1654955&r1=1654954&r2=1654955&view=diff ============================================================================== --- pig/branches/spark/test/org/apache/pig/test/TestCase.java (original) +++ pig/branches/spark/test/org/apache/pig/test/TestCase.java Tue Jan 27 02:27:45 2015 @@ -25,7 +25,6 @@ import static org.junit.Assert.fail; import java.util.List; -import org.apache.pig.ExecType; import org.apache.pig.PigServer; import org.apache.pig.builtin.mock.Storage.Data; import org.apache.pig.data.Tuple; @@ -40,7 +39,7 @@ public class TestCase { */ @Test public void testNoElseBranch() throws Exception { - PigServer pigServer = new PigServer(ExecType.LOCAL); + PigServer pigServer = new PigServer(Util.getLocalTestMode()); Data data = resetData(pigServer); data.set("foo", @@ -81,7 +80,7 @@ public class TestCase { */ @Test public void testWithElseBranch() throws Exception { - PigServer pigServer = new PigServer(ExecType.LOCAL); + PigServer pigServer = new PigServer(Util.getLocalTestMode()); Data data = resetData(pigServer); data.set("foo", @@ -122,7 +121,7 @@ public class TestCase { */ @Test public void testConditionalWithNoElse() throws Exception { - PigServer pigServer = new PigServer(ExecType.LOCAL); + PigServer pigServer = new PigServer(Util.getLocalTestMode()); Data data = resetData(pigServer); data.set("foo", @@ -163,7 +162,7 @@ public class TestCase { */ @Test public void testConditionalWithElse() throws Exception { - PigServer pigServer = new PigServer(ExecType.LOCAL); + PigServer pigServer = new PigServer(Util.getLocalTestMode()); Data data = resetData(pigServer); data.set("foo", @@ -204,7 +203,7 @@ public class TestCase { */ @Test public void testOrderOfConditions() throws Exception { - PigServer pigServer = new PigServer(ExecType.LOCAL); + PigServer pigServer = new PigServer(Util.getLocalTestMode()); Data data = resetData(pigServer); data.set("foo", @@ -244,7 +243,7 @@ public class TestCase { */ @Test public void testWithDereferenceOperator() throws Exception { - PigServer pigServer = new PigServer(ExecType.LOCAL); + PigServer pigServer = new PigServer(Util.getLocalTestMode()); Data data = resetData(pigServer); data.set("foo", @@ -281,7 +280,7 @@ public class TestCase { */ @Test(expected = FrontendException.class) public void testMissingCaseExpression() throws Exception { - PigServer pigServer = new PigServer(ExecType.LOCAL); + PigServer pigServer = new PigServer(Util.getLocalTestMode()); Data data = resetData(pigServer); data.set("foo", @@ -310,7 +309,7 @@ public class TestCase { */ @Test(expected = FrontendException.class) public void testMissingWhenExpression() throws Exception { - PigServer pigServer = new PigServer(ExecType.LOCAL); + PigServer pigServer = new PigServer(Util.getLocalTestMode()); Data data = resetData(pigServer); data.set("foo", @@ -339,7 +338,7 @@ public class TestCase { */ @Test(expected = FrontendException.class) public void testMissingThenExpression() throws Exception { - PigServer pigServer = new PigServer(ExecType.LOCAL); + PigServer pigServer = new PigServer(Util.getLocalTestMode()); Data data = resetData(pigServer); data.set("foo", @@ -368,7 +367,7 @@ public class TestCase { */ @Test(expected = FrontendException.class) public void testMissingElseExpression() throws Exception { - PigServer pigServer = new PigServer(ExecType.LOCAL); + PigServer pigServer = new PigServer(Util.getLocalTestMode()); Data data = resetData(pigServer); data.set("foo", Modified: pig/branches/spark/test/org/apache/pig/test/TestCommit.java URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/test/TestCommit.java?rev=1654955&r1=1654954&r2=1654955&view=diff ============================================================================== --- pig/branches/spark/test/org/apache/pig/test/TestCommit.java (original) +++ pig/branches/spark/test/org/apache/pig/test/TestCommit.java Tue Jan 27 02:27:45 2015 @@ -23,7 +23,6 @@ import static org.junit.Assert.assertTru import java.io.File; import java.util.Iterator; -import org.apache.pig.ExecType; import org.apache.pig.PigServer; import org.apache.pig.builtin.PigStorage; import org.apache.pig.data.Tuple; @@ -39,7 +38,7 @@ public class TestCommit { @Before public void setUp() throws Exception{ - pigServer = new PigServer(ExecType.LOCAL); + pigServer = new PigServer(Util.getLocalTestMode()); } @Test Modified: pig/branches/spark/test/org/apache/pig/test/TestConversions.java URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/test/TestConversions.java?rev=1654955&r1=1654954&r2=1654955&view=diff ============================================================================== --- pig/branches/spark/test/org/apache/pig/test/TestConversions.java (original) +++ pig/branches/spark/test/org/apache/pig/test/TestConversions.java Tue Jan 27 02:27:45 2015 @@ -386,6 +386,10 @@ public class TestConversions { m = ps.getLoadCaster().bytesToMap(s.getBytes(), rfs); assertNull(m); + s = "[]"; + m = ps.getLoadCaster().bytesToMap(s.getBytes(), rfs); + assertTrue(m.isEmpty()); + s = "(a,b)"; schema = Utils.getSchemaFromString("t:tuple()"); rfs = new ResourceSchema(schema).getFields()[0];
