[tez] branch master updated: TEZ-4142. TezUtils.createConfFromByteString on Configuration larger than 32MB throws com.google.protobuf.CodedInputStream exception

[jeagles]

This is an automated email from the ASF dual-hosted git repository.

jeagles pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/tez.git


The following commit(s) were added to refs/heads/master by this push:
     new 0eeef27  TEZ-4142. TezUtils.createConfFromByteString on Configuration 
larger than 32MB throws com.google.protobuf.CodedInputStream exception
0eeef27 is described below

commit 0eeef27413db97b52242878301788ac5fd8def16
Author: Jonathan Eagles <jeag...@apache.org>
AuthorDate: Mon Apr 13 09:20:10 2020 -0500

    TEZ-4142. TezUtils.createConfFromByteString on Configuration larger than 
32MB throws com.google.protobuf.CodedInputStream exception
---
 .../main/java/org/apache/tez/common/TezUtils.java  |  5 ++++-
 .../java/org/apache/tez/common/TestTezUtils.java   | 24 ++++++++++++++++++++++
 2 files changed, 28 insertions(+), 1 deletion(-)

diff --git a/tez-api/src/main/java/org/apache/tez/common/TezUtils.java 
b/tez-api/src/main/java/org/apache/tez/common/TezUtils.java
index 536eda4..51311ff 100644
--- a/tez-api/src/main/java/org/apache/tez/common/TezUtils.java
+++ b/tez-api/src/main/java/org/apache/tez/common/TezUtils.java
@@ -29,6 +29,7 @@ import java.util.Objects;
 
 import com.google.protobuf.ByteString;
 
+import com.google.protobuf.CodedInputStream;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.apache.hadoop.classification.InterfaceAudience;
@@ -110,7 +111,9 @@ public class TezUtils {
   public static Configuration createConfFromByteString(ByteString byteString) 
throws IOException {
     Objects.requireNonNull(byteString, "ByteString must be specified");
     try(SnappyInputStream uncompressIs = new 
SnappyInputStream(byteString.newInput());) {
-      DAGProtos.ConfigurationProto confProto = 
DAGProtos.ConfigurationProto.parseFrom(uncompressIs);
+      CodedInputStream in = CodedInputStream.newInstance(uncompressIs);
+      in.setSizeLimit(Integer.MAX_VALUE);
+      DAGProtos.ConfigurationProto confProto = 
DAGProtos.ConfigurationProto.parseFrom(in);
       Configuration conf = new Configuration(false);
       readConfFromPB(confProto, conf);
       return conf;
diff --git a/tez-common/src/test/java/org/apache/tez/common/TestTezUtils.java 
b/tez-common/src/test/java/org/apache/tez/common/TestTezUtils.java
index 04eb2c0..c0c3e5d 100644
--- a/tez-common/src/test/java/org/apache/tez/common/TestTezUtils.java
+++ b/tez-common/src/test/java/org/apache/tez/common/TestTezUtils.java
@@ -54,6 +54,30 @@ public class TestTezUtils {
     checkConf(conf);
   }
 
+  @Test (timeout=20000)
+  public void testByteStringToAndFromLargeConf() throws IOException {
+    Configuration conf = getConf();
+    int largeSize = 64 * 1024 * 1024;
+    final String alphaString = "ABCDEFGHIJKLMNOPQRSTUVWXYZ";
+    StringBuilder sb = new StringBuilder(largeSize);
+
+    for (int i = 0; i < largeSize; i++) {
+      int index = (int)(alphaString.length() * Math.random());
+      sb.append(alphaString.charAt(index));
+    }
+
+    String largeValue = sb.toString();
+    conf.set("testLargeValue", largeValue);
+    Assert.assertEquals(conf.size(), 7);
+    ByteString bsConf = TezUtils.createByteStringFromConf(conf);
+    conf.clear();
+    Assert.assertEquals(conf.size(), 0);
+    conf = TezUtils.createConfFromByteString(bsConf);
+    Assert.assertEquals(conf.size(), 7);
+    checkConf(conf);
+    Assert.assertEquals(conf.get("testLargeValue"), largeValue);
+  }
+
   @Test (timeout=2000)
   public void testPayloadToAndFromConf() throws IOException {
     Configuration conf = getConf();