This is an automated email from the ASF dual-hosted git repository.
prabhujoseph pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 9512c77 YARN-10884: Handle empty owners to parse log files (#3318)
9512c77 is described below
commit 9512c774c00c15e553bcf879a7abff338ce55580
Author: Swathi Chandrashekar <[email protected]>
AuthorDate: Tue Sep 7 21:37:51 2021 +0530
YARN-10884: Handle empty owners to parse log files (#3318)
* YARN 10884 : Parse log files which has empty owner
* Removing the whitespace
* Added a test case for null user
* Fixed indentations
* Fixed the indentation for test cases
Co-authored-by: Swathi C <[email protected]>
---
.../server/timeline/EntityGroupFSTimelineStore.java | 17 ++++++++++++++---
.../timeline/TestEntityGroupFSTimelineStore.java | 20 ++++++++++++++++++++
2 files changed, 34 insertions(+), 3 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/main/java/org/apache/hadoop/yarn/server/timeline/EntityGroupFSTimelineStore.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/main/java/org/apache/hadoop/yarn/server/timeline/EntityGroupFSTimelineStore.java
index c2992b4..4e5b796 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/main/java/org/apache/hadoop/yarn/server/timeline/EntityGroupFSTimelineStore.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/main/java/org/apache/hadoop/yarn/server/timeline/EntityGroupFSTimelineStore.java
@@ -134,6 +134,7 @@ public class EntityGroupFSTimelineStore extends
CompositeService
private int appCacheMaxSize = 0;
private List<TimelineEntityGroupPlugin> cacheIdPlugins;
private Map<TimelineEntityGroupId, EntityCacheItem> cachedLogs;
+ private boolean aclsEnabled;
@VisibleForTesting
@InterfaceAudience.Private
@@ -204,6 +205,8 @@ public class EntityGroupFSTimelineStore extends
CompositeService
YarnConfiguration
.TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_DONE_DIR_DEFAULT));
fs = activeRootPath.getFileSystem(conf);
+ aclsEnabled = conf.getBoolean(YarnConfiguration.YARN_ACL_ENABLE,
+ YarnConfiguration.DEFAULT_YARN_ACL_ENABLE);
CallerContext.setCurrent(
new CallerContext.Builder(ATS_V15_SERVER_DFS_CALLER_CTXT).build());
super.serviceInit(conf);
@@ -766,16 +769,24 @@ public class EntityGroupFSTimelineStore extends
CompositeService
continue;
}
String filename = statCache.getPath().getName();
+ String owner = statCache.getOwner();
+ //YARN-10884:Owner of File is set to Null on WASB Append
Operation.ATS fails to read such
+ //files as UGI cannot be constructed using Null User.To Fix
this,anonymous user is set
+ //when ACL us Disabled as the UGI is not needed there
+ if ((owner == null || owner.isEmpty()) && !aclsEnabled) {
+ LOG.debug("The owner was null when acl disabled, hence making the
owner anonymous");
+ owner = "anonymous";
+ }
// We should only update time for log files.
boolean shouldSetTime = true;
LOG.debug("scan for log file: {}", filename);
if (filename.startsWith(DOMAIN_LOG_PREFIX)) {
- addSummaryLog(attemptDirName, filename, statCache.getOwner(),
true);
+ addSummaryLog(attemptDirName, filename, owner, true);
} else if (filename.startsWith(SUMMARY_LOG_PREFIX)) {
- addSummaryLog(attemptDirName, filename, statCache.getOwner(),
+ addSummaryLog(attemptDirName, filename, owner,
false);
} else if (filename.startsWith(ENTITY_LOG_PREFIX)) {
- addDetailLog(attemptDirName, filename, statCache.getOwner());
+ addDetailLog(attemptDirName, filename, owner);
} else {
shouldSetTime = false;
}
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/test/java/org/apache/hadoop/yarn/server/timeline/TestEntityGroupFSTimelineStore.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/test/java/org/apache/hadoop/yarn/server/timeline/TestEntityGroupFSTimelineStore.java
index 7302ae1..fdd4772 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/test/java/org/apache/hadoop/yarn/server/timeline/TestEntityGroupFSTimelineStore.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/test/java/org/apache/hadoop/yarn/server/timeline/TestEntityGroupFSTimelineStore.java
@@ -24,6 +24,7 @@ import org.apache.hadoop.fs.FileContext;
import org.apache.hadoop.fs.FileContextTestHelper;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.metrics2.lib.MutableCounterLong;
@@ -67,6 +68,8 @@ import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
public class TestEntityGroupFSTimelineStore extends TimelineStoreTestUtils {
@@ -280,6 +283,23 @@ public class TestEntityGroupFSTimelineStore extends
TimelineStoreTestUtils {
}
@Test
+ public void testWithAnonymousUser() throws Exception {
+ try {
+ TimelineDataManager tdm =
PluginStoreTestUtils.getTdmWithMemStore(config);
+ EntityGroupFSTimelineStore.AppLogs appLogs =
+ store.new AppLogs(mainTestAppId, mainTestAppDirPath,
+ AppState.COMPLETED);
+ FileStatus fileStatus = mock(FileStatus.class);
+ when(fileStatus.getOwner()).thenReturn(null);
+ appLogs.scanForLogs();
+ appLogs.parseSummaryLogs(tdm);
+ PluginStoreTestUtils.verifyTestEntities(tdm);
+ } catch (IllegalArgumentException ie) {
+ Assert.fail("No exception needs to be thrown as anonymous user is
configured");
+ }
+ }
+
+ @Test
public void testCleanLogs() throws Exception {
// Create test dirs and files
// Irrelevant file, should not be reclaimed
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
