[tez] branch branch-0.9 updated: TEZ-4142. TezUtils.createConfFromByteString on Configuration larger than 32MB throws com.google.protobuf.CodedInputStream exception

[jeagles]

This is an automated email from the ASF dual-hosted git repository.

jeagles pushed a commit to branch branch-0.9
in repository https://gitbox.apache.org/repos/asf/tez.git


The following commit(s) were added to refs/heads/branch-0.9 by this push:
     new fb56a09  TEZ-4142. TezUtils.createConfFromByteString on Configuration 
larger than 32MB throws com.google.protobuf.CodedInputStream exception
fb56a09 is described below

commit fb56a09205a5d671e9e7aac30b2464fd344bd6a2
Author: Jonathan Eagles <jeag...@apache.org>
AuthorDate: Mon Apr 13 09:24:01 2020 -0500

    TEZ-4142. TezUtils.createConfFromByteString on Configuration larger than 
32MB throws com.google.protobuf.CodedInputStream exception
---
 .../main/java/org/apache/tez/common/TezUtils.java  |  5 ++++-
 .../java/org/apache/tez/common/TestTezUtils.java   | 24 ++++++++++++++++++++++
 2 files changed, 28 insertions(+), 1 deletion(-)

diff --git a/tez-api/src/main/java/org/apache/tez/common/TezUtils.java 
b/tez-api/src/main/java/org/apache/tez/common/TezUtils.java
index 50cac15..cf7cdf3 100644
--- a/tez-api/src/main/java/org/apache/tez/common/TezUtils.java
+++ b/tez-api/src/main/java/org/apache/tez/common/TezUtils.java
@@ -32,6 +32,7 @@ import java.util.Objects;
 
 import com.google.protobuf.ByteString;
 
+import com.google.protobuf.CodedInputStream;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.apache.hadoop.classification.InterfaceAudience;
@@ -114,7 +115,9 @@ public class TezUtils {
     // SnappyInputStream uncompressIs = new
     // SnappyInputStream(byteString.newInput());
     try(InflaterInputStream uncompressIs = new 
InflaterInputStream(byteString.newInput())) {
-      DAGProtos.ConfigurationProto confProto = 
DAGProtos.ConfigurationProto.parseFrom(uncompressIs);
+      CodedInputStream in = CodedInputStream.newInstance(uncompressIs);
+      in.setSizeLimit(Integer.MAX_VALUE);
+      DAGProtos.ConfigurationProto confProto = 
DAGProtos.ConfigurationProto.parseFrom(in);
       Configuration conf = new Configuration(false);
       readConfFromPB(confProto, conf);
       return conf;
diff --git a/tez-common/src/test/java/org/apache/tez/common/TestTezUtils.java 
b/tez-common/src/test/java/org/apache/tez/common/TestTezUtils.java
index 04eb2c0..c0c3e5d 100644
--- a/tez-common/src/test/java/org/apache/tez/common/TestTezUtils.java
+++ b/tez-common/src/test/java/org/apache/tez/common/TestTezUtils.java
@@ -54,6 +54,30 @@ public class TestTezUtils {
     checkConf(conf);
   }
 
+  @Test (timeout=20000)
+  public void testByteStringToAndFromLargeConf() throws IOException {
+    Configuration conf = getConf();
+    int largeSize = 64 * 1024 * 1024;
+    final String alphaString = "ABCDEFGHIJKLMNOPQRSTUVWXYZ";
+    StringBuilder sb = new StringBuilder(largeSize);
+
+    for (int i = 0; i < largeSize; i++) {
+      int index = (int)(alphaString.length() * Math.random());
+      sb.append(alphaString.charAt(index));
+    }
+
+    String largeValue = sb.toString();
+    conf.set("testLargeValue", largeValue);
+    Assert.assertEquals(conf.size(), 7);
+    ByteString bsConf = TezUtils.createByteStringFromConf(conf);
+    conf.clear();
+    Assert.assertEquals(conf.size(), 0);
+    conf = TezUtils.createConfFromByteString(bsConf);
+    Assert.assertEquals(conf.size(), 7);
+    checkConf(conf);
+    Assert.assertEquals(conf.get("testLargeValue"), largeValue);
+  }
+
   @Test (timeout=2000)
   public void testPayloadToAndFromConf() throws IOException {
     Configuration conf = getConf();