Repository: parquet-mr Updated Branches: refs/heads/master 83406b73e -> 454fc3655
PARQUET-342: Updates to be Java 6 compatible Author: Nezih Yigitbasi <[email protected]> Closes #248 from nezihyigitbasi/java6-fixes and squashes the following commits: 2ab2598 [Nezih Yigitbasi] Updates to be Java 6 compatible Project: http://git-wip-us.apache.org/repos/asf/parquet-mr/repo Commit: http://git-wip-us.apache.org/repos/asf/parquet-mr/commit/454fc365 Tree: http://git-wip-us.apache.org/repos/asf/parquet-mr/tree/454fc365 Diff: http://git-wip-us.apache.org/repos/asf/parquet-mr/diff/454fc365 Branch: refs/heads/master Commit: 454fc3655509f1f4f47ce44acaff7c1566ede108 Parents: 83406b7 Author: Nezih Yigitbasi <[email protected]> Authored: Tue Jul 28 14:55:14 2015 -0700 Committer: Alex Levenson <[email protected]> Committed: Tue Jul 28 14:55:14 2015 -0700 ---------------------------------------------------------------------- .../src/main/java/org/apache/parquet/Files.java | 51 ++++++++++++++++++++ .../org/apache/parquet/SemanticVersion.java | 16 ++++-- .../TestInputOutputFormatWithPadding.java | 11 ++--- .../hadoop/example/TestInputOutputFormat.java | 9 ++-- 4 files changed, 73 insertions(+), 14 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/454fc365/parquet-common/src/main/java/org/apache/parquet/Files.java ---------------------------------------------------------------------- diff --git a/parquet-common/src/main/java/org/apache/parquet/Files.java b/parquet-common/src/main/java/org/apache/parquet/Files.java new file mode 100644 index 0000000..1d2b506 --- /dev/null +++ b/parquet-common/src/main/java/org/apache/parquet/Files.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.parquet; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStreamReader; +import java.nio.charset.Charset; +import java.util.ArrayList; +import java.util.List; + +//TODO: Use java.nio.file.Files when Parquet is updated to Java 7 +public final class Files { + private Files() { } + + public static List<String> readAllLines(File file, Charset charset) throws IOException { + BufferedReader reader = new BufferedReader(new InputStreamReader(new FileInputStream(file), charset)); + try { + List<String> result = new ArrayList<String>(); + for (;;) { + String line = reader.readLine(); + if (line == null) + break; + result.add(line); + } + return result; + } + finally { + reader.close(); + } + } +} http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/454fc365/parquet-common/src/main/java/org/apache/parquet/SemanticVersion.java ---------------------------------------------------------------------- diff --git a/parquet-common/src/main/java/org/apache/parquet/SemanticVersion.java b/parquet-common/src/main/java/org/apache/parquet/SemanticVersion.java index d573a81..c6cb406 100644 --- a/parquet-common/src/main/java/org/apache/parquet/SemanticVersion.java +++ b/parquet-common/src/main/java/org/apache/parquet/SemanticVersion.java @@ -96,22 +96,30 @@ public final class SemanticVersion implements Comparable<SemanticVersion> { public int compareTo(SemanticVersion o) { int cmp; - cmp = Integer.compare(major, o.major); + cmp = compareIntegers(major, o.major); if (cmp != 0) { return cmp; } - cmp = Integer.compare(minor, o.minor); + cmp = compareIntegers(minor, o.minor); if (cmp != 0) { return cmp; } - cmp = Integer.compare(patch, o.patch); + cmp = compareIntegers(patch, o.patch); if (cmp != 0) { return cmp; } - return Boolean.compare(o.prerelease, prerelease); + return compareBooleans(o.prerelease, prerelease); + } + + int compareIntegers(int x, int y) { + return (x < y) ? -1 : ((x == y) ? 0 : 1); + } + + int compareBooleans(boolean x, boolean y) { + return (x == y) ? 0 : (x ? 1 : -1); } @Override http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/454fc365/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/TestInputOutputFormatWithPadding.java ---------------------------------------------------------------------- diff --git a/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/TestInputOutputFormatWithPadding.java b/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/TestInputOutputFormatWithPadding.java index dcb0c59..0ac9c0f 100644 --- a/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/TestInputOutputFormatWithPadding.java +++ b/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/TestInputOutputFormatWithPadding.java @@ -19,7 +19,6 @@ package org.apache.parquet.hadoop; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.io.Text; @@ -28,6 +27,7 @@ import org.apache.hadoop.mapreduce.JobContext; import org.apache.hadoop.mapreduce.Mapper; import org.apache.hadoop.mapreduce.lib.input.TextInputFormat; import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat; +import org.apache.parquet.Files; import org.apache.parquet.example.data.Group; import org.apache.parquet.example.data.simple.SimpleGroupFactory; import org.apache.parquet.format.converter.ParquetMetadataConverter; @@ -42,13 +42,11 @@ import org.junit.Assert; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; + import java.io.File; import java.io.FileOutputStream; import java.io.IOException; -import java.net.URI; import java.nio.charset.Charset; -import java.nio.file.Files; -import java.nio.file.Paths; import java.util.UUID; import static java.lang.Thread.sleep; @@ -66,6 +64,8 @@ public class TestInputOutputFormatWithPadding { .required(BINARY).as(UTF8).named("char") .named("FormatTestObject"); + private static final Charset UTF_8 = Charset.forName("UTF-8"); + /** * ParquetInputFormat that will not split the input file (easier validation) */ @@ -179,8 +179,7 @@ public class TestInputOutputFormatWithPadding { Assert.assertNotNull("Should find a data file", dataFile); StringBuilder contentBuilder = new StringBuilder(); - for (String line : Files.readAllLines( - Paths.get(dataFile.toURI()), Charset.forName("UTF-8"))) { + for (String line : Files.readAllLines(dataFile, UTF_8)) { contentBuilder.append(line); } String reconstructed = contentBuilder.toString(); http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/454fc365/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/example/TestInputOutputFormat.java ---------------------------------------------------------------------- diff --git a/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/example/TestInputOutputFormat.java b/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/example/TestInputOutputFormat.java index 987554e..5ca041b 100644 --- a/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/example/TestInputOutputFormat.java +++ b/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/example/TestInputOutputFormat.java @@ -29,7 +29,6 @@ import java.io.FileReader; import java.io.IOException; import java.lang.reflect.Method; import java.nio.charset.Charset; -import java.nio.file.Files; import java.util.Collections; import java.util.HashMap; import java.util.Iterator; @@ -48,6 +47,7 @@ import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.mapreduce.Mapper; import org.apache.hadoop.mapreduce.lib.input.TextInputFormat; import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat; +import org.apache.parquet.Files; import org.apache.parquet.Strings; import org.apache.parquet.filter2.predicate.FilterApi; import org.junit.Before; @@ -68,6 +68,7 @@ import org.apache.parquet.schema.MessageTypeParser; public class TestInputOutputFormat { private static final Log LOG = Log.getLog(TestInputOutputFormat.class); + private static final Charset UTF_8 = Charset.forName("UTF-8"); final Path parquetPath = new Path("target/test/example/TestInputOutputFormat/parquet"); final Path inputPath = new Path("src/test/java/org/apache/parquet/hadoop/example/TestInputOutputFormat.java"); final Path outputPath = new Path("target/test/example/TestInputOutputFormat/out"); @@ -258,7 +259,7 @@ public class TestInputOutputFormat { put(ParquetInputFormat.FILTER_PREDICATE, fpString); }}); - List<String> lines = Files.readAllLines(new File(outputPath.toString(), "part-m-00000").toPath(), Charset.forName("UTF-8")); + List<String> lines = Files.readAllLines(new File(outputPath.toString(), "part-m-00000"), UTF_8); assertTrue(lines.isEmpty()); } @@ -276,7 +277,7 @@ public class TestInputOutputFormat { put(ParquetInputFormat.FILTER_PREDICATE, fpString); }}); - List<String> expected = Files.readAllLines(new File(inputPath.toString()).toPath(), Charset.forName("UTF-8")); + List<String> expected = Files.readAllLines(new File(inputPath.toString()), UTF_8); // grab the lines that contain the first 500 characters (including the rest of the line past 500 characters) int size = 0; @@ -293,7 +294,7 @@ public class TestInputOutputFormat { } // put the output back into it's original format (remove the character counts / tabs) - List<String> found = Files.readAllLines(new File(outputPath.toString(), "part-m-00000").toPath(), Charset.forName("UTF-8")); + List<String> found = Files.readAllLines(new File(outputPath.toString(), "part-m-00000"), UTF_8); StringBuilder sbFound = new StringBuilder(); for (String line : found) { sbFound.append(line.split("\t", -1)[1]);
