Repository: crunch Updated Branches: refs/heads/master a8ffc4643 -> ef939b778
CRUNCH-465: Fix Avro-related build problems. Update Avro to 1.7.7 and fix resulting errors. Fix a brittle integration test. Signed-off-by: Josh Wills <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/crunch/repo Commit: http://git-wip-us.apache.org/repos/asf/crunch/commit/ef939b77 Tree: http://git-wip-us.apache.org/repos/asf/crunch/tree/ef939b77 Diff: http://git-wip-us.apache.org/repos/asf/crunch/diff/ef939b77 Branch: refs/heads/master Commit: ef939b77827ea8fdf3687e4e11f992451ad585e6 Parents: a8ffc46 Author: Matthias Friedrich <[email protected]> Authored: Tue Aug 26 21:39:55 2014 +0200 Committer: Josh Wills <[email protected]> Committed: Tue Aug 26 15:32:01 2014 -0700 ---------------------------------------------------------------------- .../it/java/org/apache/crunch/StageResultsCountersIT.java | 8 ++++---- .../java/org/apache/crunch/types/avro/AvroOutputFormat.java | 3 +-- pom.xml | 2 +- 3 files changed, 6 insertions(+), 7 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/crunch/blob/ef939b77/crunch-core/src/it/java/org/apache/crunch/StageResultsCountersIT.java ---------------------------------------------------------------------- diff --git a/crunch-core/src/it/java/org/apache/crunch/StageResultsCountersIT.java b/crunch-core/src/it/java/org/apache/crunch/StageResultsCountersIT.java index ffcc931..04711e4 100644 --- a/crunch-core/src/it/java/org/apache/crunch/StageResultsCountersIT.java +++ b/crunch-core/src/it/java/org/apache/crunch/StageResultsCountersIT.java @@ -17,8 +17,9 @@ */ package org.apache.crunch; -import static junit.framework.Assert.assertEquals; import static junit.framework.Assert.assertTrue; +import static org.hamcrest.Matchers.is; +import static org.junit.Assert.assertThat; import java.util.HashSet; import java.util.List; @@ -38,6 +39,7 @@ import org.junit.After; import org.junit.Rule; import org.junit.Test; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.Maps; import com.google.common.collect.Sets; @@ -87,9 +89,7 @@ public class StageResultsCountersIT { Map<String, Long> keywordsMap = countersToMap(result.getStageResults(), KEYWORDS_COUNTER_GROUP); - assertEquals(3, keywordsMap.size()); - - assertEquals("{NOT=157, AND=596, OR=81}", keywordsMap.toString()); + assertThat(keywordsMap, is((Map<String, Long>) ImmutableMap.of("NOT", 157L, "AND", 596L, "OR", 81L))); } private static PipelineResult coutSpecialKeywords(Pipeline pipeline, String inputFileName, PTypeFamily tf) { http://git-wip-us.apache.org/repos/asf/crunch/blob/ef939b77/crunch-core/src/main/java/org/apache/crunch/types/avro/AvroOutputFormat.java ---------------------------------------------------------------------- diff --git a/crunch-core/src/main/java/org/apache/crunch/types/avro/AvroOutputFormat.java b/crunch-core/src/main/java/org/apache/crunch/types/avro/AvroOutputFormat.java index 6dbb6de..79736b8 100644 --- a/crunch-core/src/main/java/org/apache/crunch/types/avro/AvroOutputFormat.java +++ b/crunch-core/src/main/java/org/apache/crunch/types/avro/AvroOutputFormat.java @@ -22,7 +22,6 @@ import java.io.IOException; import org.apache.avro.Schema; import org.apache.avro.file.CodecFactory; import org.apache.avro.file.DataFileWriter; -import org.apache.avro.io.DatumWriter; import org.apache.avro.mapred.AvroJob; import org.apache.avro.mapred.AvroWrapper; import org.apache.hadoop.conf.Configuration; @@ -53,7 +52,7 @@ public class AvroOutputFormat<T> extends FileOutputFormat<AvroWrapper<T>, NullWr if (org.apache.hadoop.mapred.FileOutputFormat.getCompressOutput(jc)) { int level = conf.getInt(org.apache.avro.mapred.AvroOutputFormat.DEFLATE_LEVEL_KEY, - org.apache.avro.mapred.AvroOutputFormat.DEFAULT_DEFLATE_LEVEL); + org.apache.avro.file.CodecFactory.DEFAULT_DEFLATE_LEVEL); String codecName = conf.get(AvroJob.OUTPUT_CODEC, org.apache.avro.file.DataFileConstants.DEFLATE_CODEC); CodecFactory codec = codecName.equals(org.apache.avro.file.DataFileConstants.DEFLATE_CODEC) http://git-wip-us.apache.org/repos/asf/crunch/blob/ef939b77/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index 209e067..4714b89 100644 --- a/pom.xml +++ b/pom.xml @@ -74,7 +74,7 @@ under the License. <commons-httpclient.version>3.0.1</commons-httpclient.version> <commons-logging.version>1.1.1</commons-logging.version> <commons-cli.version>1.2</commons-cli.version> - <avro.version>1.7.4</avro.version> + <avro.version>1.7.7</avro.version> <hive.version>0.13.1</hive.version> <parquet.version>1.3.2</parquet.version> <javassist.version>3.16.1-GA</javassist.version>
