TEZ-2936. Create ATS implementation that enables support for YARN-4265 (ATSv1.5) (hitesh)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/42b61f43 Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/42b61f43 Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/42b61f43 Branch: refs/heads/master Commit: 42b61f43f1e4e0cb47a55249fb21378b62bde2c4 Parents: 65c4dc6 Author: Hitesh Shah <[email protected]> Authored: Mon Mar 14 16:36:43 2016 -0700 Committer: Hitesh Shah <[email protected]> Committed: Mon Mar 14 16:36:43 2016 -0700 ---------------------------------------------------------------------- BUILDING.txt | 6 +- CHANGES.txt | 1 + pom.xml | 5 + .../java/org/apache/tez/client/TezClient.java | 51 ++- .../apache/tez/dag/api/TezConfiguration.java | 11 + .../tez/dag/history/logging/EntityTypes.java | 29 ++ .../tez/dag/history/logging/EntityTypes.java | 29 -- tez-dist/pom.xml | 3 +- tez-plugins/pom.xml | 14 + .../findbugs-exclude.xml | 16 + .../tez-yarn-timeline-cache-plugin/pom.xml | 74 ++++ .../logging/ats/TimelineCachePluginImpl.java | 166 ++++++++ .../main/javadoc/resources/META-INF/LICENSE.txt | 202 ++++++++++ .../main/javadoc/resources/META-INF/NOTICE.txt | 6 + .../src/main/resources/META-INF/LICENSE.txt | 202 ++++++++++ .../src/main/resources/META-INF/NOTICE.txt | 6 + .../ats/TestTimelineCachePluginImpl.java | 260 ++++++++++++ .../findbugs-exclude.xml | 16 + .../tez-yarn-timeline-history-with-fs/pom.xml | 149 +++++++ .../ats/acls/ATSV15HistoryACLPolicyManager.java | 263 ++++++++++++ .../ats/ATSV15HistoryLoggingService.java | 400 +++++++++++++++++++ .../main/javadoc/resources/META-INF/LICENSE.txt | 202 ++++++++++ .../main/javadoc/resources/META-INF/NOTICE.txt | 6 + .../src/main/resources/META-INF/LICENSE.txt | 202 ++++++++++ .../src/main/resources/META-INF/NOTICE.txt | 6 + .../dag/history/ats/acls/TestATSHistoryV15.java | 297 ++++++++++++++ tez-tools/analyzers/pom.xml | 9 + 27 files changed, 2582 insertions(+), 49 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tez/blob/42b61f43/BUILDING.txt ---------------------------------------------------------------------- diff --git a/BUILDING.txt b/BUILDING.txt index 241b7a8..c251b4a 100644 --- a/BUILDING.txt +++ b/BUILDING.txt @@ -74,9 +74,13 @@ To skip Tests and java docs $ mvn package -Dhadoop.version=3.0.0-SNAPSHOT -DskipTests -Dmaven.javadoc.skip=true -However, to build against hadoop versions lower than 2.6.0, you will need to do the +However, to build against hadoop versions lower or higher than 2.6.0, you will need to do the following: +For Hadoop version X where X >= 2.8.0 + + $ mvn package -Dhadoop.version=${X} -Phadoop28 -P\!hadoop26 + For Hadoop version X where 2.4.0 <= X < 2.6.0 $ mvn package -Dhadoop.version=${X} -Phadoop24 -P\!hadoop26 http://git-wip-us.apache.org/repos/asf/tez/blob/42b61f43/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index 2c87be5..c45ce37 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -7,6 +7,7 @@ INCOMPATIBLE CHANGES TEZ-3029. Add an onError method to service plugin contexts. ALL CHANGES: + TEZ-2936. Create ATS implementation that enables support for YARN-4265 (ATS v1.5) TEZ-3148. Invalid event TA_TEZ_EVENT_UPDATE on TaskAttempt. TEZ-3105. TezMxBeanResourceCalculator does not work on IBM JDK 7 or 8 causing Tez failures. TEZ-3155. Support a way to submit DAGs to a session where the DAG plan exceeds hadoop ipc limits. http://git-wip-us.apache.org/repos/asf/tez/blob/42b61f43/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index 26abc24..2b6e1d4 100644 --- a/pom.xml +++ b/pom.xml @@ -498,6 +498,11 @@ </dependency> <dependency> <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-yarn-server-timeline-pluginstorage</artifactId> + <version>${hadoop.version}</version> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-yarn-server-tests</artifactId> <version>${hadoop.version}</version> <type>test-jar</type> http://git-wip-us.apache.org/repos/asf/tez/blob/42b61f43/tez-api/src/main/java/org/apache/tez/client/TezClient.java ---------------------------------------------------------------------- diff --git a/tez-api/src/main/java/org/apache/tez/client/TezClient.java b/tez-api/src/main/java/org/apache/tez/client/TezClient.java index a6a256c..bded5a2 100644 --- a/tez-api/src/main/java/org/apache/tez/client/TezClient.java +++ b/tez-api/src/main/java/org/apache/tez/client/TezClient.java @@ -144,6 +144,10 @@ public class TezClient { "org.apache.tez.dag.history.logging.ats.ATSHistoryLoggingService"; private static final String atsHistoryACLManagerClassName = "org.apache.tez.dag.history.ats.acls.ATSHistoryACLPolicyManager"; + private static final String atsv15HistoryLoggingServiceClassName = + "org.apache.tez.dag.history.logging.ats.ATSV15HistoryLoggingService"; + private static final String atsV15HistoryACLManagerClassName = + "org.apache.tez.dag.history.ats.acls.ATSV15HistoryACLPolicyManager"; private TezClient(String name, TezConfiguration tezConf) { this(name, tezConf, tezConf.getBoolean( @@ -364,23 +368,35 @@ public class TezClient { ///need additional check for historyACLPolicyManager because tests could stub historyACLPolicyManager ///before tezclient start. If there is already a stubbed historyACLPolicyManager, we don't overwrite it - if (this.amConfig.getTezConfiguration().get( - TezConfiguration.TEZ_HISTORY_LOGGING_SERVICE_CLASS, "") - .equals(atsHistoryLoggingServiceClassName) && (historyACLPolicyManager == null)) { - LOG.info("Using " + atsHistoryACLManagerClassName + " to manage Timeline ACLs"); - try { - historyACLPolicyManager = ReflectionUtils.createClazzInstance( - atsHistoryACLManagerClassName); - historyACLPolicyManager.setConf(this.amConfig.getYarnConfiguration()); - } catch (TezReflectionException e) { - if (!amConfig.getTezConfiguration().getBoolean( - TezConfiguration.TEZ_AM_ALLOW_DISABLED_TIMELINE_DOMAINS, - TezConfiguration.TEZ_AM_ALLOW_DISABLED_TIMELINE_DOMAINS_DEFAULT)) { - LOG.warn("Could not instantiate object for " + atsHistoryACLManagerClassName - + ". ACLs cannot be enforced correctly for history data in Timeline", e); - throw e; + if (historyACLPolicyManager == null) { + //TODO: FIXME: The ACL manager should be retrieved either from the + //logging service directly or via a pluggable factory that can + //instantiate ACL managers and logging services + String logSvcClassName = amConfig.getTezConfiguration().get( + TezConfiguration.TEZ_HISTORY_LOGGING_SERVICE_CLASS, ""); + String aclMgrClassName = null; + if (logSvcClassName.equals(atsHistoryLoggingServiceClassName)) { + aclMgrClassName = atsHistoryACLManagerClassName; + } else if (logSvcClassName.equals( + atsv15HistoryLoggingServiceClassName)) { + aclMgrClassName = atsV15HistoryACLManagerClassName; + } + if (aclMgrClassName != null) { + LOG.info("Using " + aclMgrClassName + " to manage Timeline ACLs"); + try { + historyACLPolicyManager = ReflectionUtils.createClazzInstance( + aclMgrClassName); + historyACLPolicyManager.setConf(this.amConfig.getYarnConfiguration()); + } catch (TezReflectionException e) { + if (!amConfig.getTezConfiguration().getBoolean( + TezConfiguration.TEZ_AM_ALLOW_DISABLED_TIMELINE_DOMAINS, + TezConfiguration.TEZ_AM_ALLOW_DISABLED_TIMELINE_DOMAINS_DEFAULT)) { + LOG.warn("Could not instantiate object for " + aclMgrClassName + + ". ACLs cannot be enforced correctly for history data in Timeline", e); + throw e; + } + historyACLPolicyManager = null; } - historyACLPolicyManager = null; } } @@ -500,7 +516,8 @@ public class TezClient { TezConfiguration dagClientConf = new TezConfiguration(amConfig.getTezConfiguration()); Map<String, String> aclConfigs = null; - // TEZ_AM_HISTORY_LOGGING_ENABLED is a config setting enable/disable logging of all dags within a session + // TEZ_AM_HISTORY_LOGGING_ENABLED is a config setting enable/disable logging of all + // dags within a session boolean sessionHistoryLoggingEnabled = amConfig.getTezConfiguration().getBoolean( TezConfiguration.TEZ_AM_HISTORY_LOGGING_ENABLED, TezConfiguration.TEZ_AM_HISTORY_LOGGING_ENABLED_DEFAULT); http://git-wip-us.apache.org/repos/asf/tez/blob/42b61f43/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java ---------------------------------------------------------------------- diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java index 0221e6b..4fc8322 100644 --- a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java +++ b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java @@ -1545,4 +1545,15 @@ public class TezConfiguration extends Configuration { TEZ_PREFIX + "test.recovery-service-class"; @Private public static final String TEZ_AM_RECOVERY_SERVICE_CLASS_DEFAULT = "org.apache.tez.dag.history.recovery.RecoveryService"; + + /** + * Boolean value. Default false. + * By default, configured values for the Summary Entity Types for Timeline will + * not be respected and be overridden by the Timeline History Service. + */ + @Private + @ConfigurationScope(Scope.AM) + public static final String TEZ_AM_ATS_V15_OVERRIDE_SUMMARY_TYPES = + TEZ_PREFIX + "am.ats.v15.override.summary-types"; + public static final boolean TEZ_AM_ATS_V15_OVERRIDE_SUMMARY_TYPES_DEFAULT = true; } http://git-wip-us.apache.org/repos/asf/tez/blob/42b61f43/tez-common/src/main/java/org/apache/tez/dag/history/logging/EntityTypes.java ---------------------------------------------------------------------- diff --git a/tez-common/src/main/java/org/apache/tez/dag/history/logging/EntityTypes.java b/tez-common/src/main/java/org/apache/tez/dag/history/logging/EntityTypes.java new file mode 100644 index 0000000..e2f0882 --- /dev/null +++ b/tez-common/src/main/java/org/apache/tez/dag/history/logging/EntityTypes.java @@ -0,0 +1,29 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tez.dag.history.logging; + +public enum EntityTypes { + TEZ_APPLICATION, + TEZ_APPLICATION_ATTEMPT, + TEZ_CONTAINER_ID, + TEZ_DAG_ID, + TEZ_VERTEX_ID, + TEZ_TASK_ID, + TEZ_TASK_ATTEMPT_ID, +} http://git-wip-us.apache.org/repos/asf/tez/blob/42b61f43/tez-dag/src/main/java/org/apache/tez/dag/history/logging/EntityTypes.java ---------------------------------------------------------------------- diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/logging/EntityTypes.java b/tez-dag/src/main/java/org/apache/tez/dag/history/logging/EntityTypes.java deleted file mode 100644 index e2f0882..0000000 --- a/tez-dag/src/main/java/org/apache/tez/dag/history/logging/EntityTypes.java +++ /dev/null @@ -1,29 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.tez.dag.history.logging; - -public enum EntityTypes { - TEZ_APPLICATION, - TEZ_APPLICATION_ATTEMPT, - TEZ_CONTAINER_ID, - TEZ_DAG_ID, - TEZ_VERTEX_ID, - TEZ_TASK_ID, - TEZ_TASK_ATTEMPT_ID, -} http://git-wip-us.apache.org/repos/asf/tez/blob/42b61f43/tez-dist/pom.xml ---------------------------------------------------------------------- diff --git a/tez-dist/pom.xml b/tez-dist/pom.xml index 9ff4416..3201d4f 100644 --- a/tez-dist/pom.xml +++ b/tez-dist/pom.xml @@ -103,7 +103,6 @@ <artifactId>tez-yarn-timeline-history-with-acls</artifactId> <version>${project.version}</version> </dependency> - <!-- <dependency> <groupId>org.apache.tez</groupId> <artifactId>tez-yarn-timeline-history-with-fs</artifactId> @@ -113,7 +112,7 @@ <groupId>org.apache.tez</groupId> <artifactId>tez-yarn-timeline-cache-plugin</artifactId> <version>${project.version}</version> - </dependency> --> + </dependency> <dependency> <groupId>org.apache.tez</groupId> <artifactId>hadoop-shim-2.8</artifactId> http://git-wip-us.apache.org/repos/asf/tez/blob/42b61f43/tez-plugins/pom.xml ---------------------------------------------------------------------- diff --git a/tez-plugins/pom.xml b/tez-plugins/pom.xml index 6d1c88a..06a7f61 100644 --- a/tez-plugins/pom.xml +++ b/tez-plugins/pom.xml @@ -50,6 +50,20 @@ <module>tez-history-parser</module> </modules> </profile> + <profile> + <id>hadoop28</id> + <activation> + <activeByDefault>false</activeByDefault> + </activation> + <modules> + <module>tez-yarn-timeline-history</module> + <module>tez-yarn-timeline-history-with-acls</module> + <module>tez-yarn-timeline-cache-plugin</module> + <module>tez-yarn-timeline-history-with-fs</module> + <module>tez-history-parser</module> + </modules> + </profile> + </profiles> <build> http://git-wip-us.apache.org/repos/asf/tez/blob/42b61f43/tez-plugins/tez-yarn-timeline-cache-plugin/findbugs-exclude.xml ---------------------------------------------------------------------- diff --git a/tez-plugins/tez-yarn-timeline-cache-plugin/findbugs-exclude.xml b/tez-plugins/tez-yarn-timeline-cache-plugin/findbugs-exclude.xml new file mode 100644 index 0000000..5b11308 --- /dev/null +++ b/tez-plugins/tez-yarn-timeline-cache-plugin/findbugs-exclude.xml @@ -0,0 +1,16 @@ +<!-- + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. See accompanying LICENSE file. +--> +<FindBugsFilter> + +</FindBugsFilter> http://git-wip-us.apache.org/repos/asf/tez/blob/42b61f43/tez-plugins/tez-yarn-timeline-cache-plugin/pom.xml ---------------------------------------------------------------------- diff --git a/tez-plugins/tez-yarn-timeline-cache-plugin/pom.xml b/tez-plugins/tez-yarn-timeline-cache-plugin/pom.xml new file mode 100644 index 0000000..2948e96 --- /dev/null +++ b/tez-plugins/tez-yarn-timeline-cache-plugin/pom.xml @@ -0,0 +1,74 @@ +<!-- + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. See accompanying LICENSE file. +--> + +<project xmlns="http://maven.apache.org/POM/4.0.0"; + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <modelVersion>4.0.0</modelVersion> + <parent> + <groupId>org.apache.tez</groupId> + <artifactId>tez-plugins</artifactId> + <version>0.8.3-SNAPSHOT</version> + </parent> + <artifactId>tez-yarn-timeline-cache-plugin</artifactId> + + <dependencies> + <dependency> + <groupId>org.apache.tez</groupId> + <artifactId>tez-common</artifactId> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-yarn-server-timeline-pluginstorage</artifactId> + </dependency> + <dependency> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + <scope>test</scope> + </dependency> + </dependencies> + + <build> + <plugins> + <plugin> + <groupId>org.apache.rat</groupId> + <artifactId>apache-rat-plugin</artifactId> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-assembly-plugin</artifactId> + <configuration> + <descriptorRefs> + <descriptorRef>jar-with-dependencies</descriptorRef> + </descriptorRefs> + <archive> + <manifest> + <addClasspath>true</addClasspath> + </manifest> + </archive> + </configuration> + <executions> + <execution> + <id>assemble-all</id> + <phase>package</phase> + <goals> + <goal>single</goal> + </goals> + </execution> + </executions> + </plugin> + </plugins> + </build> + +</project> http://git-wip-us.apache.org/repos/asf/tez/blob/42b61f43/tez-plugins/tez-yarn-timeline-cache-plugin/src/main/java/org/apache/tez/dag/history/logging/ats/TimelineCachePluginImpl.java ---------------------------------------------------------------------- diff --git a/tez-plugins/tez-yarn-timeline-cache-plugin/src/main/java/org/apache/tez/dag/history/logging/ats/TimelineCachePluginImpl.java b/tez-plugins/tez-yarn-timeline-cache-plugin/src/main/java/org/apache/tez/dag/history/logging/ats/TimelineCachePluginImpl.java new file mode 100644 index 0000000..d81f56a --- /dev/null +++ b/tez-plugins/tez-yarn-timeline-cache-plugin/src/main/java/org/apache/tez/dag/history/logging/ats/TimelineCachePluginImpl.java @@ -0,0 +1,166 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p/> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p/> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tez.dag.history.logging.ats; + +import java.util.Collection; +import java.util.HashSet; +import java.util.Set; +import java.util.SortedSet; + +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.timeline.TimelineEntityGroupId; +import org.apache.hadoop.yarn.server.timeline.NameValuePair; +import org.apache.hadoop.yarn.server.timeline.TimelineEntityGroupPlugin; +import org.apache.tez.dag.history.logging.EntityTypes; +import org.apache.tez.dag.records.TezDAGID; +import org.apache.tez.dag.records.TezTaskAttemptID; +import org.apache.tez.dag.records.TezTaskID; +import org.apache.tez.dag.records.TezVertexID; + +import com.google.common.collect.Sets; + +public class TimelineCachePluginImpl extends TimelineEntityGroupPlugin { + + private static Set<String> summaryEntityTypes; + private static Set<String> knownEntityTypes; + + static { + knownEntityTypes = Sets.newHashSet( + EntityTypes.TEZ_DAG_ID.name(), + EntityTypes.TEZ_VERTEX_ID.name(), + EntityTypes.TEZ_TASK_ID.name(), + EntityTypes.TEZ_TASK_ATTEMPT_ID.name(), + EntityTypes.TEZ_CONTAINER_ID.name()); + summaryEntityTypes = Sets.newHashSet( + EntityTypes.TEZ_DAG_ID.name(), + EntityTypes.TEZ_APPLICATION_ATTEMPT.name(), + EntityTypes.TEZ_APPLICATION.name()); + } + + // Empty public constructor + public TimelineCachePluginImpl() { + } + + private TimelineEntityGroupId convertToTimelineEntityGroupId(String entityType, String entityId) { + if (entityType == null || entityType.isEmpty() + || entityId == null || entityId.isEmpty()) { + return null; + } + if (entityType.equals(EntityTypes.TEZ_DAG_ID.name())) { + TezDAGID dagId = TezDAGID.fromString(entityId); + if (dagId != null) { + return TimelineEntityGroupId.newInstance(dagId.getApplicationId(), dagId.toString()); + } + } else if (entityType.equals(EntityTypes.TEZ_VERTEX_ID.name())) { + TezVertexID vertexID = TezVertexID.fromString(entityId); + if (vertexID != null) { + return TimelineEntityGroupId.newInstance(vertexID.getDAGId().getApplicationId(), + vertexID.getDAGId().toString()); + } + + } else if (entityType.equals(EntityTypes.TEZ_TASK_ID.name())) { + TezTaskID taskID = TezTaskID.fromString(entityId); + if (taskID != null) { + return TimelineEntityGroupId.newInstance(taskID.getVertexID().getDAGId().getApplicationId(), + taskID.getVertexID().getDAGId().toString()); + } + } else if (entityType.equals(EntityTypes.TEZ_TASK_ATTEMPT_ID.name())) { + TezTaskAttemptID taskAttemptID = TezTaskAttemptID.fromString(entityId); + if (taskAttemptID != null) { + return TimelineEntityGroupId.newInstance( + taskAttemptID.getTaskID().getVertexID().getDAGId().getApplicationId(), + taskAttemptID.getTaskID().getVertexID().getDAGId().toString()); + } + } else if (entityType.equals(EntityTypes.TEZ_CONTAINER_ID.name())) { + String cId = entityId; + if (cId.startsWith("tez_")) { + cId = cId.substring(4); + } + ContainerId containerId = ContainerId.fromString(cId); + if (containerId != null) { + return TimelineEntityGroupId.newInstance( + containerId.getApplicationAttemptId().getApplicationId(), + containerId.getApplicationAttemptId().getApplicationId().toString()); + } + } + return null; + } + + @Override + public Set<TimelineEntityGroupId> getTimelineEntityGroupId(String entityType, + NameValuePair primaryFilter, + Collection<NameValuePair> secondaryFilters) { + if (!knownEntityTypes.contains(entityType) + || primaryFilter == null + || !knownEntityTypes.contains(primaryFilter.getName()) + || summaryEntityTypes.contains(entityType)) { + return null; + } + TimelineEntityGroupId groupId = convertToTimelineEntityGroupId(primaryFilter.getName(), + primaryFilter.getValue().toString()); + if (groupId != null) { + TimelineEntityGroupId appGroupId = + TimelineEntityGroupId.newInstance(groupId.getApplicationId(), + groupId.getApplicationId().toString()); + return Sets.newHashSet(groupId, appGroupId); + } + return null; + } + + @Override + public Set<TimelineEntityGroupId> getTimelineEntityGroupId(String entityId, String entityType) { + if (!knownEntityTypes.contains(entityType) || summaryEntityTypes.contains(entityType)) { + return null; + } + TimelineEntityGroupId groupId = convertToTimelineEntityGroupId(entityType, entityId); + if (groupId != null) { + TimelineEntityGroupId appGroupId = + TimelineEntityGroupId.newInstance(groupId.getApplicationId(), + groupId.getApplicationId().toString()); + return Sets.newHashSet(groupId, appGroupId); + } + return null; + } + + @Override + public Set<TimelineEntityGroupId> getTimelineEntityGroupId(String entityType, + SortedSet<String> entityIds, Set<String> eventTypes) { + if (!knownEntityTypes.contains(entityType) + || summaryEntityTypes.contains(entityType) + || entityIds == null || entityIds.isEmpty()) { + return null; + } + Set<TimelineEntityGroupId> groupIds = new HashSet<TimelineEntityGroupId>(); + Set<ApplicationId> appIdSet = new HashSet<ApplicationId>(); + + for (String entityId : entityIds) { + TimelineEntityGroupId groupId = convertToTimelineEntityGroupId(entityType, entityId); + if (groupId != null) { + groupIds.add(groupId); + appIdSet.add(groupId.getApplicationId()); + } + } + for (ApplicationId appId : appIdSet) { + groupIds.add(TimelineEntityGroupId.newInstance(appId, appId.toString())); + } + return groupIds; + } + +} http://git-wip-us.apache.org/repos/asf/tez/blob/42b61f43/tez-plugins/tez-yarn-timeline-cache-plugin/src/main/javadoc/resources/META-INF/LICENSE.txt ---------------------------------------------------------------------- diff --git a/tez-plugins/tez-yarn-timeline-cache-plugin/src/main/javadoc/resources/META-INF/LICENSE.txt b/tez-plugins/tez-yarn-timeline-cache-plugin/src/main/javadoc/resources/META-INF/LICENSE.txt new file mode 100644 index 0000000..d645695 --- /dev/null +++ b/tez-plugins/tez-yarn-timeline-cache-plugin/src/main/javadoc/resources/META-INF/LICENSE.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. http://git-wip-us.apache.org/repos/asf/tez/blob/42b61f43/tez-plugins/tez-yarn-timeline-cache-plugin/src/main/javadoc/resources/META-INF/NOTICE.txt ---------------------------------------------------------------------- diff --git a/tez-plugins/tez-yarn-timeline-cache-plugin/src/main/javadoc/resources/META-INF/NOTICE.txt b/tez-plugins/tez-yarn-timeline-cache-plugin/src/main/javadoc/resources/META-INF/NOTICE.txt new file mode 100644 index 0000000..88ff448 --- /dev/null +++ b/tez-plugins/tez-yarn-timeline-cache-plugin/src/main/javadoc/resources/META-INF/NOTICE.txt @@ -0,0 +1,6 @@ +Apache Tez +Copyright (c) 2015 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + http://git-wip-us.apache.org/repos/asf/tez/blob/42b61f43/tez-plugins/tez-yarn-timeline-cache-plugin/src/main/resources/META-INF/LICENSE.txt ---------------------------------------------------------------------- diff --git a/tez-plugins/tez-yarn-timeline-cache-plugin/src/main/resources/META-INF/LICENSE.txt b/tez-plugins/tez-yarn-timeline-cache-plugin/src/main/resources/META-INF/LICENSE.txt new file mode 100644 index 0000000..d645695 --- /dev/null +++ b/tez-plugins/tez-yarn-timeline-cache-plugin/src/main/resources/META-INF/LICENSE.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. http://git-wip-us.apache.org/repos/asf/tez/blob/42b61f43/tez-plugins/tez-yarn-timeline-cache-plugin/src/main/resources/META-INF/NOTICE.txt ---------------------------------------------------------------------- diff --git a/tez-plugins/tez-yarn-timeline-cache-plugin/src/main/resources/META-INF/NOTICE.txt b/tez-plugins/tez-yarn-timeline-cache-plugin/src/main/resources/META-INF/NOTICE.txt new file mode 100644 index 0000000..88ff448 --- /dev/null +++ b/tez-plugins/tez-yarn-timeline-cache-plugin/src/main/resources/META-INF/NOTICE.txt @@ -0,0 +1,6 @@ +Apache Tez +Copyright (c) 2015 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + http://git-wip-us.apache.org/repos/asf/tez/blob/42b61f43/tez-plugins/tez-yarn-timeline-cache-plugin/src/test/java/org/apache/tez/dag/history/logging/ats/TestTimelineCachePluginImpl.java ---------------------------------------------------------------------- diff --git a/tez-plugins/tez-yarn-timeline-cache-plugin/src/test/java/org/apache/tez/dag/history/logging/ats/TestTimelineCachePluginImpl.java b/tez-plugins/tez-yarn-timeline-cache-plugin/src/test/java/org/apache/tez/dag/history/logging/ats/TestTimelineCachePluginImpl.java new file mode 100644 index 0000000..562a66e --- /dev/null +++ b/tez-plugins/tez-yarn-timeline-cache-plugin/src/test/java/org/apache/tez/dag/history/logging/ats/TestTimelineCachePluginImpl.java @@ -0,0 +1,260 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p/> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p/> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tez.dag.history.logging.ats; + +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; +import java.util.SortedSet; +import java.util.TreeSet; + +import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.timeline.TimelineEntityGroupId; +import org.apache.hadoop.yarn.server.timeline.NameValuePair; +import org.apache.tez.dag.history.logging.EntityTypes; +import org.apache.tez.dag.records.TezDAGID; +import org.apache.tez.dag.records.TezTaskAttemptID; +import org.apache.tez.dag.records.TezTaskID; +import org.apache.tez.dag.records.TezVertexID; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; + +import com.google.common.collect.Sets; + +public class TestTimelineCachePluginImpl { + + static ApplicationId appId1; + static ApplicationAttemptId appAttemptId1; + static ApplicationId appId2; + static TezDAGID dagID1; + static TezVertexID vertexID1; + static TezTaskID taskID1; + static TezTaskAttemptID attemptID1; + static TezDAGID dagID2; + static TezVertexID vertexID2; + static TezTaskID taskID2; + static TezTaskAttemptID attemptID2; + static ContainerId cId1; + static ContainerId cId2; + static Map<String, String> typeIdMap1; + static Map<String, String> typeIdMap2; + + TimelineCachePluginImpl plugin = + new TimelineCachePluginImpl(); + + @BeforeClass + public static void beforeClass() { + appId1 = ApplicationId.newInstance(1000l, 111); + appId2 = ApplicationId.newInstance(1001l, 121); + appAttemptId1 = ApplicationAttemptId.newInstance(appId1, 11); + + dagID1 = TezDAGID.getInstance(appId1, 1); + vertexID1 = TezVertexID.getInstance(dagID1, 12); + taskID1 = TezTaskID.getInstance(vertexID1, 11144); + attemptID1 = TezTaskAttemptID.getInstance(taskID1, 4); + + dagID2 = TezDAGID.getInstance(appId2, 111); + vertexID2 = TezVertexID.getInstance(dagID2, 121); + taskID2 = TezTaskID.getInstance(vertexID2, 113344); + attemptID2 = TezTaskAttemptID.getInstance(taskID2, 14); + + cId1 = ContainerId.newContainerId(appAttemptId1, 1); + cId2 = ContainerId.newContainerId(ApplicationAttemptId.newInstance(appId2, 1), 22); + + typeIdMap1 = new HashMap<String, String>(); + typeIdMap1.put(EntityTypes.TEZ_DAG_ID.name(), dagID1.toString()); + typeIdMap1.put(EntityTypes.TEZ_VERTEX_ID.name(), vertexID1.toString()); + typeIdMap1.put(EntityTypes.TEZ_TASK_ID.name(), taskID1.toString()); + typeIdMap1.put(EntityTypes.TEZ_TASK_ATTEMPT_ID.name(), attemptID1.toString()); + + typeIdMap2 = new HashMap<String, String>(); + typeIdMap2.put(EntityTypes.TEZ_DAG_ID.name(), dagID2.toString()); + typeIdMap2.put(EntityTypes.TEZ_VERTEX_ID.name(), vertexID2.toString()); + typeIdMap2.put(EntityTypes.TEZ_TASK_ID.name(), taskID2.toString()); + typeIdMap2.put(EntityTypes.TEZ_TASK_ATTEMPT_ID.name(), attemptID2.toString()); + + } + + @Test + public void testGetTimelineEntityGroupIdByPrimaryFilter() { + for (Entry<String, String> entry : typeIdMap1.entrySet()) { + NameValuePair primaryFilter = new NameValuePair(entry.getKey(), entry.getValue()); + Assert.assertNull(plugin.getTimelineEntityGroupId(EntityTypes.TEZ_APPLICATION.name(), + primaryFilter, null)); + Set<TimelineEntityGroupId> groupIds = plugin.getTimelineEntityGroupId(entry.getKey(), primaryFilter, null); + if (entry.getKey().equals(EntityTypes.TEZ_DAG_ID.name())) { + Assert.assertNull(groupIds); + continue; + } + Assert.assertEquals(2, groupIds.size()); + Iterator<TimelineEntityGroupId> iter = groupIds.iterator(); + while (iter.hasNext()) { + TimelineEntityGroupId groupId = iter.next(); + Assert.assertEquals(appId1, groupId.getApplicationId()); + Assert.assertTrue((dagID1.toString().equals(groupId.getTimelineEntityGroupId())) + || (appId1.toString().equals(groupId.getTimelineEntityGroupId()))); + } + } + } + + @Test + public void testGetTimelineEntityGroupIdById() { + for (Entry<String, String> entry : typeIdMap1.entrySet()) { + Set<TimelineEntityGroupId> groupIds = plugin.getTimelineEntityGroupId(entry.getValue(), entry.getKey()); + if (entry.getKey().equals(EntityTypes.TEZ_DAG_ID.name())) { + Assert.assertNull(groupIds); + continue; + } + Assert.assertEquals(2, groupIds.size()); + Iterator<TimelineEntityGroupId> iter = groupIds.iterator(); + while (iter.hasNext()) { + TimelineEntityGroupId groupId = iter.next(); + Assert.assertEquals(appId1, groupId.getApplicationId()); + Assert.assertTrue((dagID1.toString().equals(groupId.getTimelineEntityGroupId())) + || (appId1.toString().equals(groupId.getTimelineEntityGroupId()))); + } + } + } + + @Test + public void testGetTimelineEntityGroupIdByIds() { + for (Entry<String, String> entry : typeIdMap1.entrySet()) { + SortedSet<String> entityIds = new TreeSet<String>(); + entityIds.add(entry.getValue()); + entityIds.add(typeIdMap2.get(entry.getKey())); + Set<TimelineEntityGroupId> groupIds = plugin.getTimelineEntityGroupId(entry.getKey(), + entityIds, null); + if (entry.getKey().equals(EntityTypes.TEZ_DAG_ID.name())) { + Assert.assertNull(groupIds); + continue; + } + Assert.assertEquals(4, groupIds.size()); + int found = 0; + Iterator<TimelineEntityGroupId> iter = groupIds.iterator(); + while (iter.hasNext()) { + TimelineEntityGroupId groupId = iter.next(); + if (groupId.getApplicationId().equals(appId1) + && groupId.getTimelineEntityGroupId().equals(dagID1.toString())) { + ++found; + } else if (groupId.getApplicationId().equals(appId2) + && groupId.getTimelineEntityGroupId().equals(dagID2.toString())) { + ++found; + } else if (groupId.getApplicationId().equals(appId1) + && groupId.getTimelineEntityGroupId().equals(appId1.toString())) { + ++found; + } else if (groupId.getApplicationId().equals(appId2) + && groupId.getTimelineEntityGroupId().equals(appId2.toString())) { + ++found; + } + } + Assert.assertEquals("All groupIds not returned", 4, found); + } + } + + @Test + public void testInvalidIds() { + Assert.assertNull(plugin.getTimelineEntityGroupId(EntityTypes.TEZ_DAG_ID.name(), + vertexID1.toString())); + Assert.assertNull(plugin.getTimelineEntityGroupId(EntityTypes.TEZ_VERTEX_ID.name(), + taskID1.toString())); + Assert.assertNull(plugin.getTimelineEntityGroupId(EntityTypes.TEZ_TASK_ID.name(), + attemptID1.toString())); + Assert.assertNull(plugin.getTimelineEntityGroupId(EntityTypes.TEZ_TASK_ATTEMPT_ID.name(), + dagID1.toString())); + Assert.assertNull(plugin.getTimelineEntityGroupId("", "")); + Assert.assertNull(plugin.getTimelineEntityGroupId(null, null)); + Assert.assertNull(plugin.getTimelineEntityGroupId("adadasd", EntityTypes.TEZ_DAG_ID.name())); + } + + @Test + public void testInvalidTypeRequests() { + Assert.assertNull(plugin.getTimelineEntityGroupId(EntityTypes.TEZ_APPLICATION.name(), + appId1.toString())); + Assert.assertNull(plugin.getTimelineEntityGroupId(EntityTypes.TEZ_APPLICATION_ATTEMPT.name(), + appAttemptId1.toString())); + Assert.assertNull(plugin.getTimelineEntityGroupId(EntityTypes.TEZ_CONTAINER_ID.name(), + appId1.toString())); + + Assert.assertNull(plugin.getTimelineEntityGroupId(EntityTypes.TEZ_TASK_ID.name(), null, + new HashSet<String>())); + Assert.assertNull(plugin.getTimelineEntityGroupId(EntityTypes.TEZ_TASK_ID.name(), null, + new HashSet<NameValuePair>())); + + } + + @Test + public void testContainerIdConversion() { + + String entityType = EntityTypes.TEZ_CONTAINER_ID.name(); + SortedSet<String> entityIds = new TreeSet<String>(); + entityIds.add("tez_" + cId1.toString()); + entityIds.add("tez_" + cId2.toString()); + Set<TimelineEntityGroupId> groupIds = plugin.getTimelineEntityGroupId(entityType, + entityIds, null); + Assert.assertEquals(2, groupIds.size()); + int found = 0; + Iterator<TimelineEntityGroupId> iter = groupIds.iterator(); + while (iter.hasNext()) { + TimelineEntityGroupId groupId = iter.next(); + if (groupId.getApplicationId().equals(appId1) + && groupId.getTimelineEntityGroupId().equals(appId1.toString())) { + ++found; + } else if (groupId.getApplicationId().equals(appId2) + && groupId.getTimelineEntityGroupId().equals(appId2.toString())) { + ++found; + } + } + Assert.assertEquals("All groupIds not returned", 2, found); + + groupIds.clear(); + groupIds = plugin.getTimelineEntityGroupId(cId1.toString(), entityType); + Assert.assertEquals(1, groupIds.size()); + found = 0; + iter = groupIds.iterator(); + while (iter.hasNext()) { + TimelineEntityGroupId groupId = iter.next(); + if (groupId.getApplicationId().equals(appId1) + && groupId.getTimelineEntityGroupId().equals(appId1.toString())) { + ++found; + } + } + Assert.assertEquals("All groupIds not returned", 1, found); + + groupIds.clear(); + groupIds = plugin.getTimelineEntityGroupId("tez_" + cId2.toString(), entityType); + Assert.assertEquals(1, groupIds.size()); + found = 0; + iter = groupIds.iterator(); + while (iter.hasNext()) { + TimelineEntityGroupId groupId = iter.next(); + if (groupId.getApplicationId().equals(appId2) + && groupId.getTimelineEntityGroupId().equals(appId2.toString())) { + ++found; + } + } + Assert.assertEquals("All groupIds not returned", 1, found); + + } +} http://git-wip-us.apache.org/repos/asf/tez/blob/42b61f43/tez-plugins/tez-yarn-timeline-history-with-fs/findbugs-exclude.xml ---------------------------------------------------------------------- diff --git a/tez-plugins/tez-yarn-timeline-history-with-fs/findbugs-exclude.xml b/tez-plugins/tez-yarn-timeline-history-with-fs/findbugs-exclude.xml new file mode 100644 index 0000000..5b11308 --- /dev/null +++ b/tez-plugins/tez-yarn-timeline-history-with-fs/findbugs-exclude.xml @@ -0,0 +1,16 @@ +<!-- + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. See accompanying LICENSE file. +--> +<FindBugsFilter> + +</FindBugsFilter> http://git-wip-us.apache.org/repos/asf/tez/blob/42b61f43/tez-plugins/tez-yarn-timeline-history-with-fs/pom.xml ---------------------------------------------------------------------- diff --git a/tez-plugins/tez-yarn-timeline-history-with-fs/pom.xml b/tez-plugins/tez-yarn-timeline-history-with-fs/pom.xml new file mode 100644 index 0000000..23a7f5d --- /dev/null +++ b/tez-plugins/tez-yarn-timeline-history-with-fs/pom.xml @@ -0,0 +1,149 @@ +<!-- + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. See accompanying LICENSE file. +--> + +<project xmlns="http://maven.apache.org/POM/4.0.0"; + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <modelVersion>4.0.0</modelVersion> + <parent> + <groupId>org.apache.tez</groupId> + <artifactId>tez-plugins</artifactId> + <version>0.8.3-SNAPSHOT</version> + </parent> + <artifactId>tez-yarn-timeline-history-with-fs</artifactId> + + <dependencies> + <dependency> + <groupId>org.apache.tez</groupId> + <artifactId>tez-api</artifactId> + </dependency> + <dependency> + <groupId>org.apache.tez</groupId> + <artifactId>tez-common</artifactId> + </dependency> + <dependency> + <groupId>org.apache.tez</groupId> + <artifactId>tez-dag</artifactId> + </dependency> + <dependency> + <groupId>org.apache.tez</groupId> + <artifactId>tez-yarn-timeline-history</artifactId> + </dependency> + <dependency> + <groupId>org.apache.tez</groupId> + <artifactId>tez-yarn-timeline-history</artifactId> + <type>test-jar</type> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.apache.tez</groupId> + <artifactId>tez-tests</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.apache.tez</groupId> + <artifactId>tez-tests</artifactId> + <type>test-jar</type> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-common</artifactId> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-yarn-api</artifactId> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-yarn-common</artifactId> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-yarn-client</artifactId> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-yarn-server-tests</artifactId> + <scope>test</scope> + <type>test-jar</type> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-hdfs</artifactId> + <scope>test</scope> + <type>test-jar</type> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-common</artifactId> + <scope>test</scope> + <type>test-jar</type> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-mapreduce-client-core</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-mapreduce-client-common</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-mapreduce-client-jobclient</artifactId> + <scope>test</scope> + <type>test-jar</type> + </dependency> + <dependency> + <groupId>org.codehaus.jettison</groupId> + <artifactId>jettison</artifactId> + </dependency> + <dependency> + <groupId>com.google.guava</groupId> + <artifactId>guava</artifactId> + </dependency> + <dependency> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-api</artifactId> + </dependency> + <dependency> + <groupId>org.mockito</groupId> + <artifactId>mockito-all</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>com.sun.jersey</groupId> + <artifactId>jersey-json</artifactId> + <scope>test</scope> + </dependency> + </dependencies> + + <build> + <plugins> + <plugin> + <groupId>org.apache.rat</groupId> + <artifactId>apache-rat-plugin</artifactId> + </plugin> + </plugins> + </build> + +</project> + http://git-wip-us.apache.org/repos/asf/tez/blob/42b61f43/tez-plugins/tez-yarn-timeline-history-with-fs/src/main/java/org/apache/tez/dag/history/ats/acls/ATSV15HistoryACLPolicyManager.java ---------------------------------------------------------------------- diff --git a/tez-plugins/tez-yarn-timeline-history-with-fs/src/main/java/org/apache/tez/dag/history/ats/acls/ATSV15HistoryACLPolicyManager.java b/tez-plugins/tez-yarn-timeline-history-with-fs/src/main/java/org/apache/tez/dag/history/ats/acls/ATSV15HistoryACLPolicyManager.java new file mode 100644 index 0000000..f0ec1eb --- /dev/null +++ b/tez-plugins/tez-yarn-timeline-history-with-fs/src/main/java/org/apache/tez/dag/history/ats/acls/ATSV15HistoryACLPolicyManager.java @@ -0,0 +1,263 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tez.dag.history.ats.acls; + +import java.io.IOException; +import java.util.Collections; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.service.Service; +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; +import org.apache.hadoop.yarn.api.records.timeline.TimelineEntityGroupId; +import org.apache.hadoop.yarn.api.records.timeline.TimelineDomain; +import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity; +import org.apache.hadoop.yarn.client.api.TimelineClient; +import org.apache.tez.common.security.ACLConfigurationParser; +import org.apache.tez.common.security.ACLManager; +import org.apache.tez.common.security.ACLType; +import org.apache.tez.common.security.DAGAccessControls; +import org.apache.tez.common.security.HistoryACLPolicyManager; +import org.apache.tez.common.security.HistoryACLPolicyException; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.tez.dag.api.TezConfiguration; +import org.apache.tez.dag.api.TezUncheckedException; + +public class ATSV15HistoryACLPolicyManager implements HistoryACLPolicyManager { + + private final static Logger LOG = LoggerFactory.getLogger(ATSV15HistoryACLPolicyManager.class); + + TimelineClient timelineClient; + Configuration conf; + String user; + final static String DOMAIN_ID_PREFIX = "Tez_ATS_"; + private static final String atsHistoryLoggingServiceClassName = + "org.apache.tez.dag.history.logging.ats.ATSV15HistoryLoggingService"; + + private void initializeTimelineClient() { + if (this.conf == null) { + throw new TezUncheckedException("ATSACLManager not configured"); + } + if (timelineClient != null) { + this.timelineClient.stop(); + this.timelineClient = null; + } + if (conf.getBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, + YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ENABLED)) { + this.timelineClient = TimelineClient.createTimelineClient(); + this.timelineClient.init(this.conf); + this.timelineClient.start(); + } else { + this.timelineClient = null; + if (conf.get(TezConfiguration.TEZ_HISTORY_LOGGING_SERVICE_CLASS, "") + .equals(atsHistoryLoggingServiceClassName)) { + LOG.warn(atsHistoryLoggingServiceClassName + + " is disabled due to Timeline Service being disabled, " + + YarnConfiguration.TIMELINE_SERVICE_ENABLED + " set to false"); + } + } + try { + this.user = UserGroupInformation.getCurrentUser().getShortUserName(); + } catch (IOException e) { + throw new TezUncheckedException("Unable to get Current User UGI", e); + } + } + + private String getMergedViewACLs(ACLConfigurationParser parser, + DAGAccessControls dagAccessControls) { + Map<ACLType, Set<String>> allowedUsers = parser.getAllowedUsers(); + Map<ACLType, Set<String>> allowedGroups = parser.getAllowedGroups(); + + Set<String> viewUsers = new HashSet<String>(); + viewUsers.add(user); + if (allowedUsers.containsKey(ACLType.AM_VIEW_ACL)) { + viewUsers.addAll(allowedUsers.get(ACLType.AM_VIEW_ACL)); + } + if (dagAccessControls != null && dagAccessControls.getUsersWithViewACLs() != null) { + viewUsers.addAll(dagAccessControls.getUsersWithViewACLs()); + } + + if (viewUsers.contains(ACLManager.WILDCARD_ACL_VALUE)) { + return ACLManager.WILDCARD_ACL_VALUE; + } + + Set<String> viewGroups = new HashSet<String>(); + if (allowedGroups.containsKey(ACLType.AM_VIEW_ACL)) { + viewGroups.addAll(allowedGroups.get(ACLType.AM_VIEW_ACL)); + } + if (dagAccessControls != null && dagAccessControls.getGroupsWithViewACLs() != null) { + viewGroups.addAll(dagAccessControls.getGroupsWithViewACLs()); + } + + return ACLManager.toCommaSeparatedString(viewUsers) + " " + + ACLManager.toCommaSeparatedString(viewGroups); + } + + private void createTimelineDomain(ApplicationId applicationId, + String domainId, Configuration tezConf, DAGAccessControls dagAccessControls) + throws IOException, HistoryACLPolicyException { + TimelineDomain timelineDomain = new TimelineDomain(); + timelineDomain.setId(domainId); + + ACLConfigurationParser parser = new ACLConfigurationParser(tezConf, false); + timelineDomain.setReaders(getMergedViewACLs(parser, dagAccessControls)); + timelineDomain.setWriters(user); + + // Use dummy app attempt id + ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(applicationId, 1); + try { + if (timelineClient != null) { + timelineClient.putDomain(appAttemptId, timelineDomain); + } + } catch (Exception e) { + LOG.warn("Could not post timeline domain", e); + throw new + HistoryACLPolicyException("Fail to create ACL-related domain in Timeline", e); + } + } + + + private Map<String, String> createSessionDomain(Configuration tezConf, + ApplicationId applicationId, DAGAccessControls dagAccessControls) + throws IOException, HistoryACLPolicyException { + String domainId = + tezConf.get(TezConfiguration.YARN_ATS_ACL_SESSION_DOMAIN_ID); + if (!tezConf.getBoolean(TezConfiguration.TEZ_AM_ACLS_ENABLED, + TezConfiguration.TEZ_AM_ACLS_ENABLED_DEFAULT)) { + if (domainId != null) { + throw new TezUncheckedException("ACLs disabled but DomainId is specified" + + ", aclsEnabled=true, domainId=" + domainId); + } + return null; + } + + boolean autoCreateDomain = tezConf.getBoolean(TezConfiguration.YARN_ATS_ACL_DOMAINS_AUTO_CREATE, + TezConfiguration.YARN_ATS_ACL_DOMAINS_AUTO_CREATE_DEFAULT); + + if (domainId != null) { + // do nothing + LOG.info("Using specified domainId with Timeline, domainId=" + domainId); + return null; + } else { + if (!autoCreateDomain) { + // Error - Cannot fallback to default as that leaves ACLs open + throw new TezUncheckedException("Timeline DomainId is not specified and auto-create" + + " Domains is disabled"); + } + domainId = DOMAIN_ID_PREFIX + applicationId.toString(); + createTimelineDomain(applicationId, domainId, tezConf, dagAccessControls); + LOG.info("Created Timeline Domain for History ACLs, domainId=" + domainId); + return Collections.singletonMap(TezConfiguration.YARN_ATS_ACL_SESSION_DOMAIN_ID, domainId); + } + } + + private Map<String, String> createDAGDomain(Configuration tezConf, + ApplicationId applicationId, String dagName, DAGAccessControls dagAccessControls) + throws IOException, HistoryACLPolicyException { + if (dagAccessControls == null) { + // No DAG specific ACLs + return null; + } + + String domainId = + tezConf.get(TezConfiguration.YARN_ATS_ACL_DAG_DOMAIN_ID); + if (!tezConf.getBoolean(TezConfiguration.TEZ_AM_ACLS_ENABLED, + TezConfiguration.TEZ_AM_ACLS_ENABLED_DEFAULT)) { + if (domainId != null) { + throw new TezUncheckedException("ACLs disabled but domainId for DAG is specified" + + ", aclsEnabled=true, domainId=" + domainId); + } + return null; + } + + boolean autoCreateDomain = tezConf.getBoolean(TezConfiguration.YARN_ATS_ACL_DOMAINS_AUTO_CREATE, + TezConfiguration.YARN_ATS_ACL_DOMAINS_AUTO_CREATE_DEFAULT); + + if (domainId != null) { + // do nothing + LOG.info("Using specified domainId with Timeline, domainId=" + domainId); + return null; + } else { + if (!autoCreateDomain) { + // Error - Cannot fallback to default as that leaves ACLs open + throw new TezUncheckedException("Timeline DomainId is not specified and auto-create" + + " Domains is disabled"); + } + + domainId = DOMAIN_ID_PREFIX + applicationId.toString() + "_" + dagName; + createTimelineDomain(applicationId, domainId, tezConf, dagAccessControls); + LOG.info("Created Timeline Domain for DAG-specific History ACLs, domainId=" + domainId); + return Collections.singletonMap(TezConfiguration.YARN_ATS_ACL_DAG_DOMAIN_ID, domainId); + } + } + + + @Override + public void setConf(Configuration conf) { + this.conf = conf; + initializeTimelineClient(); + } + + @Override + public Configuration getConf() { + return this.conf; + } + + @Override + public Map<String, String> setupSessionACLs(Configuration conf, ApplicationId applicationId) + throws IOException, HistoryACLPolicyException { + return createSessionDomain(conf, applicationId, null); + } + + @Override + public Map<String, String> setupNonSessionACLs(Configuration conf, ApplicationId applicationId, + DAGAccessControls dagAccessControls) throws IOException, HistoryACLPolicyException { + return createSessionDomain(conf, applicationId, dagAccessControls); + } + + @Override + public Map<String, String> setupSessionDAGACLs(Configuration conf, ApplicationId applicationId, + String dagName, DAGAccessControls dagAccessControls) throws IOException, HistoryACLPolicyException { + return createDAGDomain(conf, applicationId, dagName, dagAccessControls); + } + + @Override + public void updateTimelineEntityDomain(Object timelineEntity, String domainId) { + if (!(timelineEntity instanceof TimelineEntity)) { + throw new UnsupportedOperationException("Invalid object provided of type" + + timelineEntity.getClass().getName()); + } + TimelineEntity entity = (TimelineEntity) timelineEntity; + entity.setDomainId(domainId); + } + + @Override + public void close() { + if (timelineClient != null && timelineClient.isInState(Service.STATE.STARTED)) { + timelineClient.stop(); + } + } + +}
