Author: daijy
Date: Thu Jan 15 22:19:21 2015
New Revision: 1652288
URL: http://svn.apache.org/r1652288
Log:
PIG-2692: Make the Pig unit faciliities more generalizable and update javadocs
Modified:
pig/trunk/CHANGES.txt
pig/trunk/src/docs/src/documentation/content/xdocs/test.xml
pig/trunk/test/org/apache/pig/pigunit/PigTest.java
pig/trunk/test/org/apache/pig/test/pigunit/TestPigTest.java
Modified: pig/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1652288&r1=1652287&r2=1652288&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Thu Jan 15 22:19:21 2015
@@ -24,6 +24,8 @@ INCOMPATIBLE CHANGES
IMPROVEMENTS
+PIG-2692: Make the Pig unit faciliities more generalizable and update javadocs
(razsapps via daijy)
+
PIG-4379: Make RoundRobinPartitioner public (daijy)
PIG-4378: Better way to fix tez local mode test hanging (daijy)
Modified: pig/trunk/src/docs/src/documentation/content/xdocs/test.xml
URL:
http://svn.apache.org/viewvc/pig/trunk/src/docs/src/documentation/content/xdocs/test.xml?rev=1652288&r1=1652287&r2=1652288&view=diff
==============================================================================
--- pig/trunk/src/docs/src/documentation/content/xdocs/test.xml (original)
+++ pig/trunk/src/docs/src/documentation/content/xdocs/test.xml Thu Jan 15
22:19:21 2015
@@ -963,6 +963,50 @@ junit.framework.ComparisonFailure: null
at org.apache.pig.pigunit.PigTest.assertEquals(PigTest.java:272)
</source>
</section>
+
+<!-- +++++++++++++++++++++++++++++++++++++++ -->
+ <section>
+ <title>Mocking</title>
+
+ <p>Sometimes you need to mock out the data in specific aliases. Using
PigTest's
+ mocking you can override an alias everywhere it is assigned. If you do
not know
+ the schema (or want to keep your test dynamic) you can use
PigTest.getAliasToSchemaMap()
+ to determine the schema. If you chose to go this route, you should
cache the map for
+ the specific script to ensure efficient execution.
+ </p>
+
+ <source>
+ @Test
+ public void testTop2Queries() {
+ String[] args = {
+ "n=2",
+ };
+
+ PigTest test = new PigTest("top_queries.pig", args);
+
+ String[] mockData = {
+ "yahoo",
+ "yahoo",
+ "yahoo",
+ "twitter",
+ "facebook",
+ "facebook",
+ "linkedin",
+ };
+
+ //You should cache the map if you can
+ String schema = test.getAliasToSchemaMap().get("data");
+ test.mockAlias("data", mockData, schema);
+
+ String[] output = {
+ "(yahoo,3)",
+ "(facebook,2)",
+ };
+
+ test.assertOutputAnyOrder("queries_limit", output);
+ }
+</source>
+ </section>
</section>
<!-- +++++++++++++++++++++++++++++++++++++++ -->
Modified: pig/trunk/test/org/apache/pig/pigunit/PigTest.java
URL:
http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/pigunit/PigTest.java?rev=1652288&r1=1652287&r2=1652288&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/pigunit/PigTest.java (original)
+++ pig/trunk/test/org/apache/pig/pigunit/PigTest.java Thu Jan 15 22:19:21 2015
@@ -18,14 +18,20 @@ import java.io.FileInputStream;
import java.io.IOException;
import java.io.PrintWriter;
import java.io.StringReader;
+import java.lang.Object;
import java.nio.MappedByteBuffer;
import java.nio.channels.FileChannel;
import java.nio.charset.Charset;
+import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Collections;
import java.util.HashMap;
import java.util.Iterator;
+import java.util.List;
import java.util.Map;
+import java.util.Set;
+import com.google.common.collect.Iterables;
import junit.framework.Assert;
import org.apache.commons.lang.StringUtils;
@@ -38,6 +44,7 @@ import org.apache.pig.data.DataType;
import org.apache.pig.data.Tuple;
import org.apache.pig.impl.PigContext;
import org.apache.pig.impl.io.FileLocalizer;
+import org.apache.pig.impl.logicalLayer.FrontendException;
import org.apache.pig.impl.logicalLayer.schema.Schema;
import org.apache.pig.pigunit.pig.PigServer;
import org.apache.pig.tools.parameters.ParseException;
@@ -254,37 +261,167 @@ public class PigTest {
aliasOverrides.remove(alias);
}
- public void assertOutput(String[] expected) throws IOException,
ParseException {
+ /**
+ * Returns a Map that has alias to it's schema.
+ *
+ * @return A map that has alias name as a key and the alias's schema as a
value
+ * @throws FrontendException If there was an error dumping the schema
+ */
+ public Map<String, String> getAliasToSchemaMap() throws FrontendException,
IOException, ParseException {
+ HashMap<String, String> aliasSchemas = new HashMap<String, String>();
registerScript();
- String alias = aliasOverrides.get("LAST_STORE_ALIAS");
+ PigServer server = getPigServer();
+ Set<String> aliasKeySet = server.getAliasKeySet();
+ for (String alias: aliasKeySet) {
+ try {
+ StringBuilder tsb = new StringBuilder();
+ Schema.stringifySchema(tsb, server.dumpSchema(alias), DataType.TUPLE,
Integer.MIN_VALUE);
+ aliasSchemas.put(alias, tsb.toString());
+ } catch (FrontendException e) {
+ /**
+ * If PigServer fails to describe a schema for an alias a
FrontendException is thrown.
+ * PigServer.getAliasKeySet() returns aliases that cannot have their
schema described.
+ * We want to skip over these particular aliases.
+ */
+ if (e.getErrorCode() == 1001) {
+ //Let's print a warning
+ System.out.println(e.getMessage());
+ } else {
+ throw e;
+ }
+ }
+ }
+ return aliasSchemas;
+ }
- assertEquals(StringUtils.join(expected, "\n"),
StringUtils.join(getAliasFromCache(alias), "\n"));
+ /**
+ * Creates a temp file and populates it with the specified mock data.
+ *
+ * @param alias alias that the temp file is for
+ * @param mockData data that is being mocked for the alias
+ * @return path to the temp file
+ */
+ private String makeMockTempFile(String alias, String[] mockData) throws
IOException {
+ //The FileLocalizer uses a random variable, but that can have collisions.
By using the current time, thread,
+ //and alias we should be able to guaratee a unique file
+ String uniqueSuffix = alias + "." + System.currentTimeMillis() + "." +
Thread.currentThread().getId();
+ //PigServer/Cluster is not initialized yet. Let's initialize it.
+ if (getPigServer() == null) {
+ getCluster();
+ }
+ String path =
FileLocalizer.getTemporaryPath(getPigServer().getPigContext(),
uniqueSuffix).toString();
+ getCluster().copyFromLocalFile(mockData, path, true);
+ return path;
}
- public void assertOutput(String alias, String[] expected) throws
IOException, ParseException {
- registerScript();
+ private String getActualResults(String alias, boolean ignoreOrder) throws
IOException, ParseException {
+ //Tuples are sortable, but we should sort it as Strings for convenience
when comparing to expected data
+ Iterator<Tuple> iterator = getAliasFromCache(alias);
+ List<String> actualResults = new ArrayList<String>();
+ while (iterator.hasNext()) {
+ actualResults.add(iterator.next().toString());
+ }
+
+ if (ignoreOrder) {
+ Collections.sort(actualResults);
+ }
+ return StringUtils.join(actualResults, "\n");
+ }
- assertEquals(StringUtils.join(expected, "\n"),
StringUtils.join(getAliasFromCache(alias), "\n"));
+ /**
+ * Allows you to mock a specific alias.
+ *
+ * This method will create a temporary file on the system that contains the
mock data. It will then change the pig
+ * script to actually replace the alias when being assigned a value with a
load file command which loads the temporary
+ * file.
+ *
+ * @param alias The alias to be mocked
+ * @param mockData The data you wished to be contained in the alias where
each element in the array is a tab-delimited
+ * @param aliasSchema The schema of the alias provided. Your mockData should
fit this schema
+ */
+ public void mockAlias(String alias, String[] mockData, String aliasSchema)
throws IOException {
+ mockAlias(alias, mockData, aliasSchema, "\\t");
}
- public void assertOutput(File expected) throws IOException, ParseException {
+ /**
+ * Allows you to mock a specific alias.
+ *
+ * This method will create a temporary file on the system that contains the
mock data. It will then change the pig
+ * script to actually replace the alias when being assigned a value with a
load file command which loads the temporary
+ * file.
+ *
+ * @param alias The alias to be mocked
+ * @param mockData String array where each element is an entry in the alias
and the line has its elements delimited
+ * by the value provided by delimiter.
+ * @param aliasSchema This is the schema of the alias being mocked
+ * @param delimiter The delimiter used to separate data in mockData
+ */
+ public void mockAlias(String alias, String[] mockData, String aliasSchema,
String delimiter) throws IOException {
+ String mockFile = makeMockTempFile(alias, mockData);
+ override(alias, String.format("%s = LOAD '%s' USING PigStorage('%s') AS
%s;", alias, mockFile, delimiter, aliasSchema));
+ }
+
+ /**
+ * Compares the expected results to the results of the last alias generated
in the script. Order does not matter
+ * and as long as the result is located in any index of expected and any
line of the output then this will pass.
+ *
+ * @param expected The expected results
+ */
+ public void assertOutputAnyOrder(String[] expected) throws IOException,
ParseException {
+ assertOutput(expected, true);
+ }
+
+ public void assertOutput(String[] expected) throws IOException,
ParseException {
+ assertOutput(expected, false);
+ }
+
+ private void assertOutput(String[] expected, boolean ignoreOrder) throws
IOException, ParseException {
registerScript();
String alias = aliasOverrides.get("LAST_STORE_ALIAS");
- assertEquals(readFile(expected).replaceAll("\r\n", "\n"),
StringUtils.join(getAliasFromCache(alias), "\n"));
+ if (ignoreOrder) {
+ Arrays.sort(expected);
+ }
+ assertEquals(StringUtils.join(expected, "\n"), getActualResults(alias,
ignoreOrder));
+ }
+
+ /**
+ * Compares the expected results to the results of the provided alias's
output. Order does not matter
+ * and as long as the result is located in any index of expected and any
line of the output then this will pass.
+ *
+ * @param alias The alias whose results we want to check
+ * @param expected The expected results
+ */
+ public void assertOutputAnyOrder(String alias, String[] expected) throws
IOException, ParseException {
+ assertOutput(alias, expected, true);
+ }
+
+ public void assertOutput(String alias, String[] expected) throws
IOException, ParseException {
+ assertOutput(alias, expected, false);
}
- public void assertOutput(String alias, File expected) throws IOException,
ParseException {
+ private void assertOutput(String alias, String[] expected, boolean
ignoreOrder) throws IOException, ParseException {
registerScript();
- assertEquals(readFile(expected).replaceAll("\r\n", "\n"),
StringUtils.join(getAliasFromCache(alias), "\n"));
+ if (ignoreOrder) {
+ Arrays.sort(expected);
+ }
+ assertEquals(StringUtils.join(expected, "\n"), getActualResults(alias,
ignoreOrder));
}
+ public void assertOutput(File expected) throws IOException, ParseException {
+ assertOutput(readFile(expected).split("(\\r\\n|\\n)"), false);
+ }
+
+ public void assertOutput(String alias, File expected) throws IOException,
ParseException {
+ assertOutput(alias, readFile(expected).split("(\\r\\n|\\n)"), false);
+ }
+
public void assertOutput(String aliasInput, String[] input, String alias,
String[] expected)
throws IOException, ParseException {
assertOutput(aliasInput, input, alias, expected, "\\t");
}
-
+
public void assertOutput(String aliasInput, String[] input, String alias,
String[] expected, String delimiter)
throws IOException, ParseException {
registerScript();
@@ -292,18 +429,15 @@ public class PigTest {
StringBuilder sb = new StringBuilder();
Schema.stringifySchema(sb, getPigServer().dumpSchema(aliasInput),
DataType.TUPLE) ;
- final String destination =
FileLocalizer.getTemporaryPath(getPigServer().getPigContext()).toString();
- getCluster().copyFromLocalFile(input, destination, true);
- override(aliasInput,
- String.format("%s = LOAD '%s' USING PigStorage('%s') AS %s;",
aliasInput, destination, delimiter, sb.toString()));
+ mockAlias(aliasInput, input, sb.toString(), delimiter);
- assertOutput(alias, expected);
+ assertOutput(alias, expected, false);
}
protected void assertEquals(String expected, String current) {
Assert.assertEquals(expected, current);
}
-
+
private static String readFile(String path) throws IOException {
return readFile(new File(path));
}
Modified: pig/trunk/test/org/apache/pig/test/pigunit/TestPigTest.java
URL:
http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/pigunit/TestPigTest.java?rev=1652288&r1=1652287&r2=1652288&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/pigunit/TestPigTest.java (original)
+++ pig/trunk/test/org/apache/pig/test/pigunit/TestPigTest.java Thu Jan 15
22:19:21 2015
@@ -17,11 +17,17 @@ package org.apache.pig.test.pigunit;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.io.PrintWriter;
+import java.lang.String;
+import java.util.HashMap;
+import java.util.Map;
+
+import junit.framework.ComparisonFailure;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
@@ -432,4 +438,170 @@ public class TestPigTest {
scriptFile.delete();
bootupFile.delete();
}
+
+ @Test
+ public void testMockedAliasWithDefaultDelimiter() throws Exception {
+ String[] args = {
+ "n=3",
+ "reducers=1",
+ "input=top_queries_input_data.txt",
+ "output=top_3_queries",
+ };
+
+ test = new PigTest(PIG_SCRIPT, args, null, null);
+
+ String[] mockData = {
+ "Apache\t99",
+ "Pig\t42",
+ "GitHub\t107",
+ "Google\t404"
+ };
+ test.mockAlias("queries_ordered", mockData,"(query: chararray,count:
int)");
+
+ String[] expectedOutput = {
+ "(Apache,99)",
+ "(Pig,42)",
+ "(GitHub,107)"
+ };
+ test.assertOutput(expectedOutput);
+ }
+
+ @Test
+ public void testMockedAliasWithDifferentDelimiter() throws Exception {
+ String[] args = {
+ "n=3",
+ "reducers=1",
+ "input=top_queries_input_data.txt",
+ "output=top_3_queries",
+ };
+
+ test = new PigTest(PIG_SCRIPT, args, null, null);
+
+ String[] mockData = {
+ "Apache,99",
+ "Pig,42",
+ "GitHub,107",
+ "Google,404"
+ };
+ test.mockAlias("queries_ordered", mockData,"(query: chararray,count:
int)", ",");
+
+ String[] expectedOutput = {
+ "(Apache,99)",
+ "(Pig,42)",
+ "(GitHub,107)"
+ };
+ test.assertOutput(expectedOutput);
+ }
+
+ @Test
+ public void testAliasSchemaMap() throws Exception {
+ String[] args = {
+ "n=3",
+ "reducers=1",
+ "input=top_queries_input_data.txt",
+ "output=top_3_queries",
+ };
+
+ test = new PigTest(PIG_SCRIPT, args, null, null);
+
+ final Map<String, String> expected = new HashMap<String, String>();
+ expected.put("data", "(query: chararray,count: int)");
+ expected.put("queries_group", "(group: chararray,data: {(query:
chararray,count: int)})");
+ expected.put("queries_sum", "(query: chararray,count: long)");
+ expected.put("queries_ordered", "(query: chararray,count: long)");
+ expected.put("queries_limit", "(query: chararray,count: long)");
+ Map<String, String> map = test.getAliasToSchemaMap();
+
+ assertEquals(expected, map);
+ }
+
+ @Test
+ public void testAnyOrderOutput() throws Exception {
+ String[] args = {
+ "n=3",
+ "reducers=1",
+ "input=top_queries_input_data.txt",
+ "output=top_3_queries",
+ };
+
+ test = new PigTest(PIG_SCRIPT, args);
+
+ String[] reorderedExpectedOutput = {
+ "(twitter,7)",
+ "(yahoo,25)",
+ "(facebook,15)"
+ };
+ test.assertOutputAnyOrder(reorderedExpectedOutput);
+ }
+
+ @Test
+ public void testAnyOrderOutputForAlias() throws Exception {
+ String[] args = {
+ "n=3",
+ "reducers=1",
+ "input=top_queries_input_data.txt",
+ "output=top_3_queries",
+ };
+
+ test = new PigTest(PIG_SCRIPT, args);
+
+ String[] reorderedExpectedOutput = {
+ "(twitter,7)",
+ "(yahoo,25)",
+ "(facebook,15)"
+ };
+ test.assertOutputAnyOrder("queries_limit", reorderedExpectedOutput);
+ }
+
+ @Test
+ public void testSpecificOrderOutput() throws Exception {
+ String[] args = {
+ "n=3",
+ "reducers=1",
+ "input=top_queries_input_data.txt",
+ "output=top_3_queries",
+ };
+
+ test = new PigTest(PIG_SCRIPT, args);
+
+ String[] reorderedExpectedOutput = {
+ "(twitter,7)",
+ "(yahoo,25)",
+ "(facebook,15)"
+ };
+
+ try {
+ test.assertOutput(reorderedExpectedOutput);
+ fail("assertOutput should fail when the records are unordered.");
+ }
+ catch(ComparisonFailure e) {
+ //Test passes
+ }
+ }
+
+ @Test
+ public void testSpecificOrderOutputForAlias() throws Exception {
+ String[] args = {
+ "n=3",
+ "reducers=1",
+ "input=top_queries_input_data.txt",
+ "output=top_3_queries",
+ };
+
+ test = new PigTest(PIG_SCRIPT, args);
+
+ String[] reorderedExpectedOutput = {
+ "(twitter,7)",
+ "(yahoo,25)",
+ "(facebook,15)"
+ };
+
+ try {
+ test.assertOutput("queries_limit", reorderedExpectedOutput);
+ fail("assertOutput should fail when the records are unordered.");
+ }
+ catch(ComparisonFailure e) {
+ //Test passes
+ }
+ }
}
