Repository: bigtop Updated Branches: refs/heads/master 09931a37b -> 7aa5fb1c0
BIGTOP-1812. Fix bigpetstore mapreduce-pig integration test w/ nested paths Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/7aa5fb1c Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/7aa5fb1c Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/7aa5fb1c Branch: refs/heads/master Commit: 7aa5fb1c08e5c6b4b73e5640332a15ab88db9f9d Parents: 09931a3 Author: jayunit100 <[email protected]> Authored: Thu Apr 9 11:20:21 2015 -0400 Committer: jayunit100 <[email protected]> Committed: Thu Apr 9 11:34:17 2015 -0400 ---------------------------------------------------------------------- .../org/apache/bigtop/bigpetstore/BigPetStoreMahoutIT.java | 5 +++-- .../org/apache/bigtop/bigpetstore/BigPetStorePigIT.java | 9 +++++---- .../java/org/apache/bigtop/bigpetstore/ITUtils.java | 5 +++-- 3 files changed, 11 insertions(+), 8 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/bigtop/blob/7aa5fb1c/bigtop-bigpetstore/bigpetstore-mapreduce/src/integrationTest/java/org/apache/bigtop/bigpetstore/BigPetStoreMahoutIT.java ---------------------------------------------------------------------- diff --git a/bigtop-bigpetstore/bigpetstore-mapreduce/src/integrationTest/java/org/apache/bigtop/bigpetstore/BigPetStoreMahoutIT.java b/bigtop-bigpetstore/bigpetstore-mapreduce/src/integrationTest/java/org/apache/bigtop/bigpetstore/BigPetStoreMahoutIT.java index b07c5a0..4791b2b 100644 --- a/bigtop-bigpetstore/bigpetstore-mapreduce/src/integrationTest/java/org/apache/bigtop/bigpetstore/BigPetStoreMahoutIT.java +++ b/bigtop-bigpetstore/bigpetstore-mapreduce/src/integrationTest/java/org/apache/bigtop/bigpetstore/BigPetStoreMahoutIT.java @@ -32,8 +32,9 @@ import com.google.common.base.Predicate; public class BigPetStoreMahoutIT { + //the cleaned path has subdirs, one for tsv, one for mahout numerical format, and so on. public static final Path INPUT_DIR_PATH = - new Path(ITUtils.BPS_TEST_PIG_CLEANED, MahoutPaths.Mahout.name()); + new Path(ITUtils.BPS_TEST_PIG_CLEANED_ROOT, MahoutPaths.Mahout.name()); public static final String INPUT_DIR_PATH_STR = INPUT_DIR_PATH.toString(); private static final Path MAHOUT_OUTPUT_DIR = createTestOutputPath(MahoutPaths.Mahout.name()); private static final Path ALS_FACTORIZATION_OUTPUT_DIR = @@ -70,4 +71,4 @@ public class BigPetStoreMahoutIT { ITUtils.assertOutput(ALS_RECOMMENDATIONS_DIR, TEST_OUTPUT_FORMAT); } -} \ No newline at end of file +} http://git-wip-us.apache.org/repos/asf/bigtop/blob/7aa5fb1c/bigtop-bigpetstore/bigpetstore-mapreduce/src/integrationTest/java/org/apache/bigtop/bigpetstore/BigPetStorePigIT.java ---------------------------------------------------------------------- diff --git a/bigtop-bigpetstore/bigpetstore-mapreduce/src/integrationTest/java/org/apache/bigtop/bigpetstore/BigPetStorePigIT.java b/bigtop-bigpetstore/bigpetstore-mapreduce/src/integrationTest/java/org/apache/bigtop/bigpetstore/BigPetStorePigIT.java index 78d5c6b..3458abd 100644 --- a/bigtop-bigpetstore/bigpetstore-mapreduce/src/integrationTest/java/org/apache/bigtop/bigpetstore/BigPetStorePigIT.java +++ b/bigtop-bigpetstore/bigpetstore-mapreduce/src/integrationTest/java/org/apache/bigtop/bigpetstore/BigPetStorePigIT.java @@ -16,7 +16,7 @@ package org.apache.bigtop.bigpetstore; import static org.apache.bigtop.bigpetstore.ITUtils.BPS_TEST_GENERATED; -import static org.apache.bigtop.bigpetstore.ITUtils.BPS_TEST_PIG_CLEANED; +import static org.apache.bigtop.bigpetstore.ITUtils.BPS_TEST_PIG_CLEANED_ROOT; import static org.apache.bigtop.bigpetstore.ITUtils.fs; import java.io.File; @@ -69,7 +69,7 @@ public class BigPetStorePigIT { public void setupTest() throws Throwable { ITUtils.setup(); try { - FileSystem.get(new Configuration()).delete(BPS_TEST_PIG_CLEANED, true); + FileSystem.get(new Configuration()).delete(BPS_TEST_PIG_CLEANED_ROOT, true); FileSystem.get(new Configuration()).delete(BPS_TEST_PIG_COUNT_PRODUCTS, true); } catch (Exception e) { throw new RuntimeException(e); @@ -78,7 +78,7 @@ public class BigPetStorePigIT { static Map<Path, Predicate<String>> TESTS = ImmutableMap.of( /** Test of the main output */ - BPS_TEST_PIG_CLEANED, ITUtils.VERIFICATION_PERDICATE, + new Path(BPS_TEST_PIG_CLEANED_ROOT,"tsv"), ITUtils.VERIFICATION_PERDICATE, // Example of how to count products after doing basic pig data cleanup BPS_TEST_PIG_COUNT_PRODUCTS, ITUtils.VERIFICATION_PERDICATE, // Test the output that is to be used as an input for Mahout. @@ -87,7 +87,8 @@ public class BigPetStorePigIT { @Test public void testPetStoreCorePipeline() throws Exception { - runPig(BPS_TEST_GENERATED, BPS_TEST_PIG_CLEANED, PIG_SCRIPT); + //outputs multiple data sets, Mahout, tsv, ... under BPS_TEST_PIG_CLEANED_ROOT + runPig(BPS_TEST_GENERATED, BPS_TEST_PIG_CLEANED_ROOT, PIG_SCRIPT); for (Entry<Path, Predicate<String>> e : TESTS.entrySet()) { ITUtils.assertOutput(e.getKey(), e.getValue()); } http://git-wip-us.apache.org/repos/asf/bigtop/blob/7aa5fb1c/bigtop-bigpetstore/bigpetstore-mapreduce/src/integrationTest/java/org/apache/bigtop/bigpetstore/ITUtils.java ---------------------------------------------------------------------- diff --git a/bigtop-bigpetstore/bigpetstore-mapreduce/src/integrationTest/java/org/apache/bigtop/bigpetstore/ITUtils.java b/bigtop-bigpetstore/bigpetstore-mapreduce/src/integrationTest/java/org/apache/bigtop/bigpetstore/ITUtils.java index fd53dc1..8f78751 100644 --- a/bigtop-bigpetstore/bigpetstore-mapreduce/src/integrationTest/java/org/apache/bigtop/bigpetstore/ITUtils.java +++ b/bigtop-bigpetstore/bigpetstore-mapreduce/src/integrationTest/java/org/apache/bigtop/bigpetstore/ITUtils.java @@ -67,9 +67,10 @@ public class ITUtils { public static final Path BPS_TEST_GENERATED = createTestOutputPath(BigPetStoreConstants.OUTPUTS.generated.name()); - public static final Path BPS_TEST_PIG_CLEANED = - createTestOutputPath (BigPetStoreConstants.OUTPUTS.cleaned.name()); + //there will be a tsv file under here... + public static final Path BPS_TEST_PIG_CLEANED_ROOT = + createTestOutputPath (BigPetStoreConstants.OUTPUTS.cleaned.name()); public static Path createTestOutputPath(String... pathParts) { Path path = TEST_OUTPUT_DIR; for(String pathPart: pathParts) {
