This is an automated email from the ASF dual-hosted git repository.
abstractdog pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/tez.git
The following commit(s) were added to refs/heads/master by this push:
new 6e357a055 TEZ-4014: Allow DAGAppMaster to read configuration from
plaintext (1/3) (#408) (Laszlo Bodor co-authored by Eric Wohlstadter reviewed
by Ayush Saxena)
6e357a055 is described below
commit 6e357a055067a4ad1ba61a51aca4ef3f0ebf9679
Author: Bodor Laszlo <bodorlaszlo0...@gmail.com>
AuthorDate: Tue Sep 2 08:35:34 2025 +0200
TEZ-4014: Allow DAGAppMaster to read configuration from plaintext (1/3)
(#408) (Laszlo Bodor co-authored by Eric Wohlstadter reviewed by Ayush Saxena)
---
pom.xml | 1 +
.../java/org/apache/tez/client/TezClientUtils.java | 15 ++++++++-
.../apache/tez/dag/api/NamedEntityDescriptor.java | 9 +++++-
.../org/apache/tez/dag/api/TezConfiguration.java | 7 +++++
.../java/org/apache/tez/dag/api/TezConstants.java | 1 +
.../java/org/apache/tez/dag/api/UserPayload.java | 13 +++++---
.../api/ContainerLauncherDescriptor.java | 6 ++++
.../api/ServicePluginsDescriptor.java | 7 +++--
.../api/TaskCommunicatorDescriptor.java | 5 +++
.../api/TaskSchedulerDescriptor.java | 6 ++++
.../org/apache/tez/common/TezUtilsInternal.java | 12 ++++++--
.../java/org/apache/tez/common/TestTezUtils.java | 36 ++++++++++++++++++++++
.../test/resources/service_plugins_descriptor.json | 26 ++++++++++++++++
tez-common/src/test/resources/tez-site.xml | 22 +++++++++++++
14 files changed, 156 insertions(+), 10 deletions(-)
diff --git a/pom.xml b/pom.xml
index a7dddbd37..f539f9af3 100644
--- a/pom.xml
+++ b/pom.xml
@@ -898,6 +898,7 @@
<artifactId>apache-rat-plugin</artifactId>
<configuration>
<excludes>
+ <exclude>**/*.json</exclude>
<exclude>CHANGES.txt</exclude>
<exclude>**/LICENSE*</exclude>
<!-- IDE files -->
diff --git a/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java
b/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java
index 8dade76cd..0a3c14fd6 100644
--- a/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java
+++ b/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java
@@ -110,6 +110,9 @@ import org.apache.tez.dag.api.records.DAGProtos.DAGPlan;
import org.apache.tez.dag.api.records.DAGProtos.PlanKeyValuePair;
import org.apache.tez.serviceplugins.api.ServicePluginsDescriptor;
+import com.fasterxml.jackson.annotation.JsonAutoDetect;
+import com.fasterxml.jackson.annotation.PropertyAccessor;
+import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Function;
import com.google.common.base.Strings;
@@ -200,6 +203,16 @@ public final class TezClientUtils {
return usingTezArchive;
}
+ public static ServicePluginsDescriptor
createPluginsDescriptorFromJSON(InputStream is) throws IOException {
+ ObjectMapper objectMapper = new ObjectMapper();
+ objectMapper.setVisibility(PropertyAccessor.FIELD,
JsonAutoDetect.Visibility.ANY);
+ if (is != null) {
+ return objectMapper.readValue(is, ServicePluginsDescriptor.class);
+ } else {
+ return ServicePluginsDescriptor.create(false);
+ }
+ }
+
private static boolean addLocalResources(Configuration conf,
String[] configUris, Map<String, LocalResource> tezJarResources,
Credentials credentials) throws IOException {
@@ -834,7 +847,7 @@ public final class TezClientUtils {
}
}
- static ConfigurationProto createFinalConfProtoForApp(Configuration amConf,
+ public static ConfigurationProto createFinalConfProtoForApp(Configuration
amConf,
ServicePluginsDescriptor servicePluginsDescriptor) {
assert amConf != null;
ConfigurationProto.Builder builder = ConfigurationProto.newBuilder();
diff --git
a/tez-api/src/main/java/org/apache/tez/dag/api/NamedEntityDescriptor.java
b/tez-api/src/main/java/org/apache/tez/dag/api/NamedEntityDescriptor.java
index 86ae26c87..db8527be3 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/NamedEntityDescriptor.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/NamedEntityDescriptor.java
@@ -23,7 +23,14 @@ import org.apache.hadoop.classification.InterfaceAudience;
@SuppressWarnings("unchecked")
public class NamedEntityDescriptor<T extends NamedEntityDescriptor<T>> extends
EntityDescriptor<NamedEntityDescriptor<T>> {
- private final String entityName;
+
+ private String entityName;
+
+ /**
+ * Public constructor to allow this descriptor to be instantiated by Jackson.
+ */
+ @InterfaceAudience.Private
+ public NamedEntityDescriptor() {}
@InterfaceAudience.Private
public NamedEntityDescriptor(String entityName, String className) {
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
index c81549cee..5baa6c152 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
@@ -2332,4 +2332,11 @@ public class TezConfiguration extends Configuration {
@ConfigurationScope(Scope.DAG)
@ConfigurationProperty
public static final String TEZ_TASK_ATTEMPT_HOOKS = TEZ_TASK_PREFIX +
"attempt.hooks";
+
+ /**
+ * Comma-separated list of additional hadoop config files to load from
CLASSPATH in standalone mode.
+ */
+ @ConfigurationScope(Scope.AM)
+ @ConfigurationProperty
+ public static final String TEZ_AM_STANDALONE_CONFS = TEZ_AM_PREFIX +
"standalone.confs";
}
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/TezConstants.java
b/tez-api/src/main/java/org/apache/tez/dag/api/TezConstants.java
index 379eb0cb1..9bf5e0503 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/TezConstants.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/TezConstants.java
@@ -49,6 +49,7 @@ public final class TezConstants {
TEZ_AM_SECURITY_SERVICE_AUTHORIZATION_CLIENT =
"security.job.client.protocol.acl";
+ public static final String SERVICE_PLUGINS_DESCRIPTOR_JSON =
"service_plugins_descriptor.json";
public static final String TEZ_PB_BINARY_CONF_NAME = "tez-conf.pb";
public static final String TEZ_PB_PLAN_BINARY_NAME = "tez-dag.pb";
public static final String TEZ_PB_PLAN_TEXT_NAME = "tez-dag.pb.txt";
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/UserPayload.java
b/tez-api/src/main/java/org/apache/tez/dag/api/UserPayload.java
index 3711b5dec..d6481a83c 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/UserPayload.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/UserPayload.java
@@ -22,7 +22,7 @@ import java.nio.ByteBuffer;
import javax.annotation.Nullable;
-import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import com.google.common.annotations.VisibleForTesting;
@@ -31,12 +31,17 @@ import com.google.common.annotations.VisibleForTesting;
* Wrapper class to hold user payloads
* Provides a version to help in evolving the payloads
*/
-@Public
+@InterfaceAudience.Public
public final class UserPayload {
- private final ByteBuffer payload;
- private final int version;
+ private ByteBuffer payload;
+ private int version;
private static final ByteBuffer EMPTY_BYTE = ByteBuffer.wrap(new byte[0]);
+ /**
+ * Public constructor to allow this descriptor to be instantiated by Jackson.
+ */
+ public UserPayload() {}
+
private UserPayload(@Nullable ByteBuffer payload) {
this(payload, 0);
}
diff --git
a/tez-api/src/main/java/org/apache/tez/serviceplugins/api/ContainerLauncherDescriptor.java
b/tez-api/src/main/java/org/apache/tez/serviceplugins/api/ContainerLauncherDescriptor.java
index ff3c90e9e..39a99cb9b 100644
---
a/tez-api/src/main/java/org/apache/tez/serviceplugins/api/ContainerLauncherDescriptor.java
+++
b/tez-api/src/main/java/org/apache/tez/serviceplugins/api/ContainerLauncherDescriptor.java
@@ -22,6 +22,12 @@ import org.apache.tez.dag.api.NamedEntityDescriptor;
@InterfaceStability.Unstable
public class ContainerLauncherDescriptor extends
NamedEntityDescriptor<ContainerLauncherDescriptor> {
+ /**
+ * Public constructor to allow this descriptor to be instantiated by Jackson.
+ */
+ @InterfaceAudience.Private
+ public ContainerLauncherDescriptor() {}
+
private ContainerLauncherDescriptor(String containerLauncherName, String
containerLauncherClassname) {
super(containerLauncherName, containerLauncherClassname);
}
diff --git
a/tez-api/src/main/java/org/apache/tez/serviceplugins/api/ServicePluginsDescriptor.java
b/tez-api/src/main/java/org/apache/tez/serviceplugins/api/ServicePluginsDescriptor.java
index 89083a0ad..fd199fe76 100644
---
a/tez-api/src/main/java/org/apache/tez/serviceplugins/api/ServicePluginsDescriptor.java
+++
b/tez-api/src/main/java/org/apache/tez/serviceplugins/api/ServicePluginsDescriptor.java
@@ -29,13 +29,16 @@ import org.apache.tez.dag.api.TezConfiguration;
@InterfaceStability.Unstable
public class ServicePluginsDescriptor {
- private final boolean enableContainers;
- private final boolean enableUber;
+ private boolean enableContainers;
+ private boolean enableUber;
private TaskSchedulerDescriptor[] taskSchedulerDescriptors;
private ContainerLauncherDescriptor[] containerLauncherDescriptors;
private TaskCommunicatorDescriptor[] taskCommunicatorDescriptors;
+ @InterfaceAudience.Private
+ public ServicePluginsDescriptor() {}
+
private ServicePluginsDescriptor(boolean enableContainers, boolean
enableUber,
TaskSchedulerDescriptor[]
taskSchedulerDescriptors,
ContainerLauncherDescriptor[]
containerLauncherDescriptors,
diff --git
a/tez-api/src/main/java/org/apache/tez/serviceplugins/api/TaskCommunicatorDescriptor.java
b/tez-api/src/main/java/org/apache/tez/serviceplugins/api/TaskCommunicatorDescriptor.java
index 57ac385fc..717887dad 100644
---
a/tez-api/src/main/java/org/apache/tez/serviceplugins/api/TaskCommunicatorDescriptor.java
+++
b/tez-api/src/main/java/org/apache/tez/serviceplugins/api/TaskCommunicatorDescriptor.java
@@ -22,6 +22,11 @@ import org.apache.tez.dag.api.NamedEntityDescriptor;
@InterfaceStability.Unstable
public class TaskCommunicatorDescriptor extends
NamedEntityDescriptor<TaskCommunicatorDescriptor> {
+ /**
+ * Public constructor to allow this descriptor to be instantiated by Jackson.
+ */
+ @InterfaceAudience.Private
+ public TaskCommunicatorDescriptor() {}
private TaskCommunicatorDescriptor(String taskCommName, String
taskCommClassname) {
super(taskCommName, taskCommClassname);
diff --git
a/tez-api/src/main/java/org/apache/tez/serviceplugins/api/TaskSchedulerDescriptor.java
b/tez-api/src/main/java/org/apache/tez/serviceplugins/api/TaskSchedulerDescriptor.java
index 12e091930..7c7cc4fae 100644
---
a/tez-api/src/main/java/org/apache/tez/serviceplugins/api/TaskSchedulerDescriptor.java
+++
b/tez-api/src/main/java/org/apache/tez/serviceplugins/api/TaskSchedulerDescriptor.java
@@ -22,6 +22,12 @@ import org.apache.tez.dag.api.NamedEntityDescriptor;
@InterfaceStability.Unstable
public class TaskSchedulerDescriptor extends
NamedEntityDescriptor<TaskSchedulerDescriptor> {
+ /**
+ * Public constructor to allow this descriptor to be instantiated by Jackson.
+ */
+ @InterfaceAudience.Private
+ public TaskSchedulerDescriptor() { }
+
private TaskSchedulerDescriptor(String taskSchedulerName, String
schedulerClassname) {
super(taskSchedulerName, schedulerClassname);
}
diff --git
a/tez-common/src/main/java/org/apache/tez/common/TezUtilsInternal.java
b/tez-common/src/main/java/org/apache/tez/common/TezUtilsInternal.java
index d7cfcfb55..661d77320 100644
--- a/tez-common/src/main/java/org/apache/tez/common/TezUtilsInternal.java
+++ b/tez-common/src/main/java/org/apache/tez/common/TezUtilsInternal.java
@@ -21,6 +21,7 @@ import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
+import java.io.InputStream;
import java.lang.management.ManagementFactory;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
@@ -74,14 +75,21 @@ public final class TezUtilsInternal {
private TezUtilsInternal() {}
- public static ConfigurationProto readUserSpecifiedTezConfiguration(String
baseDir) throws
- IOException {
+ public static ConfigurationProto readUserSpecifiedTezConfiguration(String
baseDir) throws IOException {
File confPBFile = new File(baseDir, TezConstants.TEZ_PB_BINARY_CONF_NAME);
try (FileInputStream fis = new FileInputStream(confPBFile)) {
return ConfigurationProto.parseFrom(fis);
}
}
+ public static Configuration readTezConfigurationXml(InputStream is) {
+ Configuration configuration = new Configuration();
+ if (is != null) {
+ configuration.addResource(is);
+ }
+ return configuration;
+ }
+
public static void addUserSpecifiedTezConfiguration(Configuration conf,
List<PlanKeyValuePair>
kvPairList) {
if (kvPairList != null && !kvPairList.isEmpty()) {
diff --git a/tez-common/src/test/java/org/apache/tez/common/TestTezUtils.java
b/tez-common/src/test/java/org/apache/tez/common/TestTezUtils.java
index 0d862efdd..146a2dbcf 100644
--- a/tez-common/src/test/java/org/apache/tez/common/TestTezUtils.java
+++ b/tez-common/src/test/java/org/apache/tez/common/TestTezUtils.java
@@ -17,18 +17,28 @@
package org.apache.tez.common;
+import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import java.io.IOException;
+import java.io.InputStream;
import java.util.BitSet;
import java.util.HashMap;
import java.util.Map;
import java.util.Random;
import org.apache.hadoop.conf.Configuration;
+import org.apache.tez.client.TezClientUtils;
+import org.apache.tez.dag.api.TezConfiguration;
+import org.apache.tez.dag.api.TezConstants;
import org.apache.tez.dag.api.UserPayload;
import org.apache.tez.dag.api.records.DAGProtos;
+import org.apache.tez.serviceplugins.api.ContainerLauncherDescriptor;
+import org.apache.tez.serviceplugins.api.ServicePluginsDescriptor;
+import org.apache.tez.serviceplugins.api.TaskCommunicatorDescriptor;
+import org.apache.tez.serviceplugins.api.TaskSchedulerDescriptor;
import com.google.protobuf.ByteString;
@@ -291,4 +301,30 @@ public class TestTezUtils {
assertEquals(confBuilder.getConfKeyValuesList().size(), 1);
}
+ @Test(timeout = 5000)
+ public void testReadTezConfigurationXmlFromClasspath() throws IOException {
+ InputStream is =
ClassLoader.getSystemResourceAsStream(TezConfiguration.TEZ_SITE_XML);
+ Configuration conf = TezUtilsInternal.readTezConfigurationXml(is);
+ assertEquals("tez.tar.gz", conf.get("tez.lib.uris"));
+ }
+
+ @Test(timeout = 5000)
+ public void testPluginsDescriptorFromJSON() throws IOException {
+ InputStream is =
ClassLoader.getSystemResourceAsStream(TezConstants.SERVICE_PLUGINS_DESCRIPTOR_JSON);
+ ServicePluginsDescriptor spd =
TezClientUtils.createPluginsDescriptorFromJSON(is);
+ TaskSchedulerDescriptor tsd = spd.getTaskSchedulerDescriptors()[0];
+ ContainerLauncherDescriptor cld = spd.getContainerLauncherDescriptors()[0];
+ TaskCommunicatorDescriptor tcd = spd.getTaskCommunicatorDescriptors()[0];
+
+ assertFalse(spd.areContainersEnabled());
+ assertTrue(spd.isUberEnabled());
+ assertEquals("testScheduler0_class", tsd.getClassName());
+ assertEquals("testScheduler0", tsd.getEntityName());
+ assertEquals("testLauncher0_class", cld.getClassName());
+ assertEquals("testLauncher0", cld.getEntityName());
+ assertEquals("testComm0_class", tcd.getClassName());
+ assertEquals("testComm0", tcd.getEntityName());
+ assertEquals(1, tcd.getUserPayload().getVersion());
+ assertArrayEquals(new byte[] {0, 0, 0, 1},
tcd.getUserPayload().deepCopyAsArray());
+ }
}
diff --git a/tez-common/src/test/resources/service_plugins_descriptor.json
b/tez-common/src/test/resources/service_plugins_descriptor.json
new file mode 100644
index 000000000..31eb1274c
--- /dev/null
+++ b/tez-common/src/test/resources/service_plugins_descriptor.json
@@ -0,0 +1,26 @@
+{
+ "taskSchedulerDescriptors": [
+ {
+ "className": "testScheduler0_class",
+ "entityName": "testScheduler0"
+ }
+ ],
+ "containerLauncherDescriptors": [
+ {
+ "className": "testLauncher0_class",
+ "entityName": "testLauncher0"
+ }
+ ],
+ "taskCommunicatorDescriptors": [
+ {
+ "userPayload": {
+ "payload": "AAAAAQ==",
+ "version": 1
+ },
+ "className": "testComm0_class",
+ "entityName": "testComm0"
+ }
+ ],
+ "enableContainers": false,
+ "enableUber": true
+}
diff --git a/tez-common/src/test/resources/tez-site.xml
b/tez-common/src/test/resources/tez-site.xml
new file mode 100644
index 000000000..db2b2ab3b
--- /dev/null
+++ b/tez-common/src/test/resources/tez-site.xml
@@ -0,0 +1,22 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+<configuration>
+ <property>
+ <name>tez.lib.uris</name>
+ <value>tez.tar.gz</value>
+ </property>
+</configuration>
\ No newline at end of file
