Repository: crunch Updated Branches: refs/heads/apache-crunch-0.8 67de89ab0 -> ade97e715
CRUNCH-465: Fix Avro-related build problems. Update Avro to 1.7.7 and fix resulting errors. Fix a brittle integration test. Signed-off-by: Josh Wills <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/crunch/repo Commit: http://git-wip-us.apache.org/repos/asf/crunch/commit/ade97e71 Tree: http://git-wip-us.apache.org/repos/asf/crunch/tree/ade97e71 Diff: http://git-wip-us.apache.org/repos/asf/crunch/diff/ade97e71 Branch: refs/heads/apache-crunch-0.8 Commit: ade97e715aafa2be32eac849ac493e1230fe3697 Parents: 67de89a Author: Matthias Friedrich <[email protected]> Authored: Tue Aug 26 21:39:55 2014 +0200 Committer: Josh Wills <[email protected]> Committed: Tue Aug 26 16:23:09 2014 -0700 ---------------------------------------------------------------------- .../it/java/org/apache/crunch/StageResultsCountersIT.java | 8 ++++---- .../java/org/apache/crunch/types/avro/AvroOutputFormat.java | 3 +-- pom.xml | 2 +- 3 files changed, 6 insertions(+), 7 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/crunch/blob/ade97e71/crunch-core/src/it/java/org/apache/crunch/StageResultsCountersIT.java ---------------------------------------------------------------------- diff --git a/crunch-core/src/it/java/org/apache/crunch/StageResultsCountersIT.java b/crunch-core/src/it/java/org/apache/crunch/StageResultsCountersIT.java index ffcc931..04711e4 100644 --- a/crunch-core/src/it/java/org/apache/crunch/StageResultsCountersIT.java +++ b/crunch-core/src/it/java/org/apache/crunch/StageResultsCountersIT.java @@ -17,8 +17,9 @@ */ package org.apache.crunch; -import static junit.framework.Assert.assertEquals; import static junit.framework.Assert.assertTrue; +import static org.hamcrest.Matchers.is; +import static org.junit.Assert.assertThat; import java.util.HashSet; import java.util.List; @@ -38,6 +39,7 @@ import org.junit.After; import org.junit.Rule; import org.junit.Test; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.Maps; import com.google.common.collect.Sets; @@ -87,9 +89,7 @@ public class StageResultsCountersIT { Map<String, Long> keywordsMap = countersToMap(result.getStageResults(), KEYWORDS_COUNTER_GROUP); - assertEquals(3, keywordsMap.size()); - - assertEquals("{NOT=157, AND=596, OR=81}", keywordsMap.toString()); + assertThat(keywordsMap, is((Map<String, Long>) ImmutableMap.of("NOT", 157L, "AND", 596L, "OR", 81L))); } private static PipelineResult coutSpecialKeywords(Pipeline pipeline, String inputFileName, PTypeFamily tf) { http://git-wip-us.apache.org/repos/asf/crunch/blob/ade97e71/crunch-core/src/main/java/org/apache/crunch/types/avro/AvroOutputFormat.java ---------------------------------------------------------------------- diff --git a/crunch-core/src/main/java/org/apache/crunch/types/avro/AvroOutputFormat.java b/crunch-core/src/main/java/org/apache/crunch/types/avro/AvroOutputFormat.java index 6dbb6de..79736b8 100644 --- a/crunch-core/src/main/java/org/apache/crunch/types/avro/AvroOutputFormat.java +++ b/crunch-core/src/main/java/org/apache/crunch/types/avro/AvroOutputFormat.java @@ -22,7 +22,6 @@ import java.io.IOException; import org.apache.avro.Schema; import org.apache.avro.file.CodecFactory; import org.apache.avro.file.DataFileWriter; -import org.apache.avro.io.DatumWriter; import org.apache.avro.mapred.AvroJob; import org.apache.avro.mapred.AvroWrapper; import org.apache.hadoop.conf.Configuration; @@ -53,7 +52,7 @@ public class AvroOutputFormat<T> extends FileOutputFormat<AvroWrapper<T>, NullWr if (org.apache.hadoop.mapred.FileOutputFormat.getCompressOutput(jc)) { int level = conf.getInt(org.apache.avro.mapred.AvroOutputFormat.DEFLATE_LEVEL_KEY, - org.apache.avro.mapred.AvroOutputFormat.DEFAULT_DEFLATE_LEVEL); + org.apache.avro.file.CodecFactory.DEFAULT_DEFLATE_LEVEL); String codecName = conf.get(AvroJob.OUTPUT_CODEC, org.apache.avro.file.DataFileConstants.DEFLATE_CODEC); CodecFactory codec = codecName.equals(org.apache.avro.file.DataFileConstants.DEFLATE_CODEC) http://git-wip-us.apache.org/repos/asf/crunch/blob/ade97e71/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index 8250712..2da28a3 100644 --- a/pom.xml +++ b/pom.xml @@ -73,7 +73,7 @@ under the License. <commons-httpclient.version>3.0.1</commons-httpclient.version> <commons-logging.version>1.1.1</commons-logging.version> <commons-cli.version>1.2</commons-cli.version> - <avro.version>1.7.4</avro.version> + <avro.version>1.7.7</avro.version> <parquet.version>1.3.2</parquet.version> <javassist.version>3.16.1-GA</javassist.version> <jackson.version>1.8.8</jackson.version>
