This is an automated email from the ASF dual-hosted git repository.
abhi pushed a commit to branch ranger-2.5
in repository https://gitbox.apache.org/repos/asf/ranger.git
The following commit(s) were added to refs/heads/ranger-2.5 by this push:
new 8a6c423d6 RANGER-4837: Fix flaky test by iterating over datewise
subfolders for counting orc logs (#337)
8a6c423d6 is described below
commit 8a6c423d66fd8fc6dbdd60c1ed1fc17438f9e134
Author: Fateh Singh <[email protected]>
AuthorDate: Mon Jul 15 12:49:22 2024 -0700
RANGER-4837: Fix flaky test by iterating over datewise subfolders for
counting orc logs (#337)
(cherry picked from commit c9361dc1c4b02c5271db28c59893a574983353ed)
---
.../org/apache/ranger/audit/TestAuditQueue.java | 60 ++++++++++++++--------
1 file changed, 39 insertions(+), 21 deletions(-)
diff --git
a/security-admin/src/test/java/org/apache/ranger/audit/TestAuditQueue.java
b/security-admin/src/test/java/org/apache/ranger/audit/TestAuditQueue.java
index 40dca25f5..aef14a987 100644
--- a/security-admin/src/test/java/org/apache/ranger/audit/TestAuditQueue.java
+++ b/security-admin/src/test/java/org/apache/ranger/audit/TestAuditQueue.java
@@ -27,6 +27,7 @@ import java.nio.file.Files;
import java.nio.file.Paths;
import java.time.LocalDate;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.List;
import java.util.Properties;
@@ -735,7 +736,7 @@ public class TestAuditQueue {
try {
Thread.sleep(40000);
} catch (InterruptedException e) {
- System.out.println(e);
+ logger.error(e.getMessage());
}
queue.waitToComplete();
assertTrue("File created", logFile.exists());
@@ -745,7 +746,9 @@ public class TestAuditQueue {
@Test
public void testAuditFileQueueSpoolORCRollover(){
String appType = "test";
- int messageToSend = 100000;
+ int messageToSend = 1000;
+ int preRolloverMessagesCount = (int)(0.8*messageToSend);
+ int postRolloverMessagesCount = messageToSend -
preRolloverMessagesCount;
String spoolFolderName = "target/spool";
String logFolderName = "target/testAuditFileQueueSpoolORC";
try {
@@ -760,11 +763,7 @@ public class TestAuditQueue {
}
assertTrue(Files.notExists(Paths.get(spoolFolderName)));
assertTrue(Files.notExists(Paths.get(logFolderName)));
- String subdir = appType + "/" +
LocalDate.now().toString().replace("-","");
File logFolder = new File(logFolderName);
- File logSubfolder = new File(logFolder, subdir);
- String logFileName = "test_ranger_audit.orc";
- File logFile = new File(logSubfolder, logFileName);
Properties props = new Properties();
props.put(AuditProviderFactory.AUDIT_IS_ENABLED_PROP, "true");
String hdfsPropPrefix = AuditProviderFactory.AUDIT_DEST_BASE +
".hdfs";
@@ -774,6 +773,7 @@ public class TestAuditQueue {
"%app-type%_ranger_audit.orc");
String orcPrefix = hdfsPropPrefix + ".orc";
props.put(orcPrefix+".compression","snappy");
+ //large numbers used here to ensure that file rollover happens
because of file rollover seconds and not orc file /related props
props.put(orcPrefix+".buffersize",""+100000000000000L);
props.put(orcPrefix+".stripesize",""+100000000000000L);
props.put(hdfsPropPrefix + ".batch.queuetype","filequeue");
@@ -782,35 +782,53 @@ public class TestAuditQueue {
String fileSpoolPrefix = filequeuePrefix + ".filespool";
props.put(fileSpoolPrefix+".dir",spoolFolderName);
props.put(fileSpoolPrefix+".buffer.size",""+100000000000000L);
- props.put(fileSpoolPrefix+".file.rollover.sec",""+300);
+ props.put(fileSpoolPrefix+".file.rollover.sec",""+5);
AuditProviderFactory factory = new AuditProviderFactory();
factory.init(props, appType);
AuditHandler queue = factory.getAuditProvider();
- for (int i = 0; i < messageToSend; i++) {
+ for (int i = 0; i < preRolloverMessagesCount; i++) {
queue.log(createEvent());
try {
- Thread.sleep(5);
+ Thread.sleep(10);
} catch (InterruptedException e) {
- System.out.println(e);
+ logger.error(e.getMessage());
}
}
+ //wait for rollover to happen
try {
- Thread.sleep(40000);
+ Thread.sleep(10000);
} catch (InterruptedException e) {
- System.out.println(e);
+ logger.error(e.getMessage());
+ }
+ //send some more logs
+ for (int i = 0; i < postRolloverMessagesCount; i++) {
+ queue.log(createEvent());
+ try {
+ Thread.sleep(10);
+ } catch (InterruptedException e) {
+ logger.error(e.getMessage());
+ }
}
queue.waitToComplete();
- assertTrue("File created", logFile.exists());
- File[] listOfFiles = logSubfolder.listFiles();
int totalLogsOrc = 0;
- if (listOfFiles != null){
- for(File f : listOfFiles){
- if (f.getName().endsWith(".orc")){
- totalLogsOrc +=
getOrcFileRowCount(f.getPath());
+ File appSubFolder = new File(logFolder,appType);
+ String[] datewiseSubfolders = appSubFolder.list();
+ logger.info("subfolder list="+
Arrays.toString(datewiseSubfolders));
+ if (datewiseSubfolders != null) {
+ for (String dateSubfolder : datewiseSubfolders){
+ File logSubfolder = new File(appSubFolder,
dateSubfolder);
+ File[] listOfFiles = logSubfolder.listFiles();
+ if (listOfFiles != null){
+ for(File f : listOfFiles){
+ if
(f.getName().endsWith(".orc")){
+ logger.info("Reading
orc file:"+f.getName());
+ totalLogsOrc +=
getOrcFileRowCount(f.getPath());
+ }
+ }
}
}
}
- System.out.println("Number of logs in orc="+totalLogsOrc);
+ logger.info("Number of logs in orc="+totalLogsOrc);
long totalLogsArchive = 0;
try {
@@ -826,7 +844,7 @@ public class TestAuditQueue {
} catch (IOException e) {
throw new RuntimeException(e);
}
- System.out.println("Number of logs in
archive:"+totalLogsArchive);
+ logger.info("Number of logs in archive:"+totalLogsArchive);
assertEquals(totalLogsOrc, totalLogsArchive);
long notYetConvertedToORCLogsCount = 0;
@@ -848,7 +866,7 @@ public class TestAuditQueue {
catch (IOException e){
throw new RuntimeException(e);
}
- System.out.println("Number of logs not converted to
ORC:"+notYetConvertedToORCLogsCount);
+ logger.info("Number of logs not converted to
ORC:"+notYetConvertedToORCLogsCount);
assertEquals(messageToSend,
notYetConvertedToORCLogsCount+totalLogsArchive);
}
