Repository: kudu Updated Branches: refs/heads/master 0442d47d5 -> da9f97e7f
[java] Remove private method usage in kudu-mapreduce kudu-mapreduce uses private APIs from KuduPredicate. This is an issue because the API returns a protobuf generated class and uses shaded APIs. This isnât strictly an issue in the gradle build itself because calling the shaded methods is ok, but it is an issue in IDEs like Intellij. Regardless a shaded dependency is not part of the public API and should not be exposed/used because it can cause problems like this. A follow on patch will work on enforcing that protobuf is not exposed in the client API so this doesnât happen again. Change-Id: I7ce54f715f970d1c7f9433e39af27d618e73fe6a Reviewed-on: http://gerrit.cloudera.org:8080/10424 Tested-by: Kudu Jenkins Reviewed-by: Adar Dembo <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/kudu/repo Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/a587848f Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/a587848f Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/a587848f Branch: refs/heads/master Commit: a587848f2dae241602da66abfaadb85e4c39a317 Parents: 0442d47 Author: Grant Henke <[email protected]> Authored: Wed May 16 10:28:19 2018 -0500 Committer: Grant Henke <[email protected]> Committed: Wed May 16 17:40:59 2018 +0000 ---------------------------------------------------------------------- .../org/apache/kudu/client/KuduPredicate.java | 34 ++++++++++++++++++++ .../kudu/mapreduce/KuduTableInputFormat.java | 9 ++---- .../kudu/mapreduce/KuduTableMapReduceUtil.java | 7 ++-- 3 files changed, 38 insertions(+), 12 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kudu/blob/a587848f/java/kudu-client/src/main/java/org/apache/kudu/client/KuduPredicate.java ---------------------------------------------------------------------- diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/KuduPredicate.java b/java/kudu-client/src/main/java/org/apache/kudu/client/KuduPredicate.java index 997c50b..b0ff0b4 100644 --- a/java/kudu-client/src/main/java/org/apache/kudu/client/KuduPredicate.java +++ b/java/kudu-client/src/main/java/org/apache/kudu/client/KuduPredicate.java @@ -17,6 +17,9 @@ package org.apache.kudu.client; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; import java.math.BigInteger; import java.math.BigDecimal; import java.util.ArrayList; @@ -765,6 +768,37 @@ public class KuduPredicate { } /** + * Serializes a list of {@code KuduPredicate} into a byte array. + * @return the serialized kudu predicates + * @throws IOException + */ + @InterfaceAudience.LimitedPrivate("kudu-mapreduce") + public static byte[] serialize(List<KuduPredicate> predicates) throws IOException { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + for (KuduPredicate predicate : predicates) { + Common.ColumnPredicatePB message = predicate.toPB(); + message.writeDelimitedTo(baos); + } + return baos.toByteArray(); + } + + /** + * Serializes a list of {@code KuduPredicate} into a byte array. + * @return the serialized kudu predicates + * @throws IOException + */ + @InterfaceAudience.LimitedPrivate("kudu-mapreduce") + public static List<KuduPredicate> deserialize(Schema schema, byte[] bytes) throws IOException { + ByteArrayInputStream bais = new ByteArrayInputStream(bytes); + List<KuduPredicate> predicates = new ArrayList<>(); + while (bais.available() > 0) { + Common.ColumnPredicatePB message = Common.ColumnPredicatePB.parseDelimitedFrom(bais); + predicates.add(KuduPredicate.fromPB(schema, message)); + } + return predicates; + } + + /** * Convert the predicate to the protobuf representation. * @return the protobuf message for this predicate */ http://git-wip-us.apache.org/repos/asf/kudu/blob/a587848f/java/kudu-mapreduce/src/main/java/org/apache/kudu/mapreduce/KuduTableInputFormat.java ---------------------------------------------------------------------- diff --git a/java/kudu-mapreduce/src/main/java/org/apache/kudu/mapreduce/KuduTableInputFormat.java b/java/kudu-mapreduce/src/main/java/org/apache/kudu/mapreduce/KuduTableInputFormat.java index cbf12f6..c09f805 100644 --- a/java/kudu-mapreduce/src/main/java/org/apache/kudu/mapreduce/KuduTableInputFormat.java +++ b/java/kudu-mapreduce/src/main/java/org/apache/kudu/mapreduce/KuduTableInputFormat.java @@ -251,14 +251,9 @@ public class KuduTableInputFormat extends InputFormat<NullWritable, RowResult> } } - this.predicates = new ArrayList<>(); try { - InputStream is = - new ByteArrayInputStream(Base64.decodeBase64(conf.get(ENCODED_PREDICATES_KEY, ""))); - while (is.available() > 0) { - this.predicates.add(KuduPredicate.fromPB(table.getSchema(), - Common.ColumnPredicatePB.parseDelimitedFrom(is))); - } + byte[] bytes = Base64.decodeBase64(conf.get(ENCODED_PREDICATES_KEY, "")); + this.predicates = KuduPredicate.deserialize(table.getSchema(), bytes); } catch (IOException e) { throw new RuntimeException("unable to deserialize predicates from the configuration", e); } http://git-wip-us.apache.org/repos/asf/kudu/blob/a587848f/java/kudu-mapreduce/src/main/java/org/apache/kudu/mapreduce/KuduTableMapReduceUtil.java ---------------------------------------------------------------------- diff --git a/java/kudu-mapreduce/src/main/java/org/apache/kudu/mapreduce/KuduTableMapReduceUtil.java b/java/kudu-mapreduce/src/main/java/org/apache/kudu/mapreduce/KuduTableMapReduceUtil.java index d2689ce..52b2fac 100644 --- a/java/kudu-mapreduce/src/main/java/org/apache/kudu/mapreduce/KuduTableMapReduceUtil.java +++ b/java/kudu-mapreduce/src/main/java/org/apache/kudu/mapreduce/KuduTableMapReduceUtil.java @@ -258,11 +258,8 @@ public class KuduTableMapReduceUtil { * @return the encoded predicates */ static String base64EncodePredicates(List<KuduPredicate> predicates) throws IOException { - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - for (KuduPredicate predicate : predicates) { - predicate.toPB().writeDelimitedTo(baos); - } - return Base64.encodeBase64String(baos.toByteArray()); + byte[] bytes = KuduPredicate.serialize(predicates); + return Base64.encodeBase64String(bytes); }
