This is an automated email from the ASF dual-hosted git repository. heneveld pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/brooklyn-server.git
commit 1ca2110a9e420307c0f6f930caec5987d129d73f Author: Mykola Mandra <[email protected]> AuthorDate: Fri Jul 16 16:54:59 2021 +0100 Introduce lineId for log entries Signed-off-by: Mykola Mandra <[email protected]> --- .../util/core/logbook/BrooklynLogEntry.java | 9 +++++++ .../util/core/logbook/file/FileLogStore.java | 9 +++++-- .../opensearch/BrooklynOpenSearchModel.java | 11 ++++++++ .../logbook/opensearch/OpenSearchLogStore.java | 6 ++++- .../util/core/logbook/file/FileLogStoreTest.java | 29 ++++++++++++++++------ 5 files changed, 54 insertions(+), 10 deletions(-) diff --git a/core/src/main/java/org/apache/brooklyn/util/core/logbook/BrooklynLogEntry.java b/core/src/main/java/org/apache/brooklyn/util/core/logbook/BrooklynLogEntry.java index 4ba0256..fabf8a3 100644 --- a/core/src/main/java/org/apache/brooklyn/util/core/logbook/BrooklynLogEntry.java +++ b/core/src/main/java/org/apache/brooklyn/util/core/logbook/BrooklynLogEntry.java @@ -31,6 +31,7 @@ import java.util.Date; */ public class BrooklynLogEntry { + String lineId; @JsonProperty("timestamp") String timestampString; Date datetime; @@ -43,6 +44,14 @@ public class BrooklynLogEntry { String threadName; String message; + public String getLineId() { + return lineId; + } + + public void setLineId(String lineId) { + this.lineId = lineId; + } + public String getTimestampString() { return timestampString; } diff --git a/core/src/main/java/org/apache/brooklyn/util/core/logbook/file/FileLogStore.java b/core/src/main/java/org/apache/brooklyn/util/core/logbook/file/FileLogStore.java index e3d87a8..6a8b6c3 100644 --- a/core/src/main/java/org/apache/brooklyn/util/core/logbook/file/FileLogStore.java +++ b/core/src/main/java/org/apache/brooklyn/util/core/logbook/file/FileLogStore.java @@ -39,6 +39,7 @@ import java.nio.file.Paths; import java.util.Calendar; import java.util.Date; import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Predicate; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -140,9 +141,12 @@ public class FileLogStore implements LogStore { return isLogLevelMatch && isDateTimeFromMatch && isDateTimeToMatch && isSearchPhraseMatch; }; + // Use line count to supply identification of go lines. + AtomicInteger lineCount = new AtomicInteger(); + // Filter result first, we need to know how many to skip go get the tail. List<BrooklynLogEntry> filteredQueryResult = stream - .map(this::parseLogLine) + .map(line -> parseLogLine(line, lineCount)) .filter(filter) .collect(Collectors.toList()); @@ -166,7 +170,7 @@ public class FileLogStore implements LogStore { return ImmutableList.of(); } - protected BrooklynLogEntry parseLogLine(String logLine) { + protected BrooklynLogEntry parseLogLine(String logLine, AtomicInteger lineCount) { Pattern p = Pattern.compile(this.logLinePattern); Matcher m = p.matcher(logLine); BrooklynLogEntry entry = null; @@ -185,6 +189,7 @@ public class FileLogStore implements LogStore { entry.setClazz(m.group("class")); entry.setThreadName(m.group("threadName")); entry.setMessage(m.group("message")); + entry.setLineId(String.valueOf(lineCount.incrementAndGet())); } return entry; } diff --git a/core/src/main/java/org/apache/brooklyn/util/core/logbook/opensearch/BrooklynOpenSearchModel.java b/core/src/main/java/org/apache/brooklyn/util/core/logbook/opensearch/BrooklynOpenSearchModel.java index 1d71eec..69003cd 100644 --- a/core/src/main/java/org/apache/brooklyn/util/core/logbook/opensearch/BrooklynOpenSearchModel.java +++ b/core/src/main/java/org/apache/brooklyn/util/core/logbook/opensearch/BrooklynOpenSearchModel.java @@ -77,6 +77,9 @@ class BrooklynOpenSearchModel { @JsonProperty("_type") String type; + @JsonProperty("_id") + String id; + @JsonProperty("_source") BrooklynLogEntry source; @@ -96,6 +99,14 @@ class BrooklynOpenSearchModel { this.type = type; } + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + public BrooklynLogEntry getSource() { return source; } diff --git a/core/src/main/java/org/apache/brooklyn/util/core/logbook/opensearch/OpenSearchLogStore.java b/core/src/main/java/org/apache/brooklyn/util/core/logbook/opensearch/OpenSearchLogStore.java index 05a08dd..2e813a6 100644 --- a/core/src/main/java/org/apache/brooklyn/util/core/logbook/opensearch/OpenSearchLogStore.java +++ b/core/src/main/java/org/apache/brooklyn/util/core/logbook/opensearch/OpenSearchLogStore.java @@ -186,7 +186,11 @@ public class OpenSearchLogStore implements LogStore { // Get the filtered stream from elastic search query result. List<BrooklynLogEntry> brooklynLogEntries = jsonResponse.hits.hits.stream() - .map(BrooklynOpenSearchModel.OpenSearchHit::getSource).collect(Collectors.toList()); + .map(openSearchHit -> { + BrooklynLogEntry entry = openSearchHit.getSource(); + entry.setLineId(openSearchHit.getId()); + return entry; + }).collect(Collectors.toList()); // Note, 'tail' requires to reverse the order back since elastic search requires 'sort' + 'desc' to get // last number of items, when requested. diff --git a/core/src/test/java/org/apache/brooklyn/util/core/logbook/file/FileLogStoreTest.java b/core/src/test/java/org/apache/brooklyn/util/core/logbook/file/FileLogStoreTest.java index 1d5308e..de9d962 100644 --- a/core/src/test/java/org/apache/brooklyn/util/core/logbook/file/FileLogStoreTest.java +++ b/core/src/test/java/org/apache/brooklyn/util/core/logbook/file/FileLogStoreTest.java @@ -31,6 +31,7 @@ import java.io.File; import java.util.List; import java.util.Objects; import java.util.TimeZone; +import java.util.concurrent.atomic.AtomicInteger; import static org.apache.brooklyn.util.core.logbook.file.FileLogStore.LOGBOOK_LOG_STORE_DATEFORMAT; import static org.apache.brooklyn.util.core.logbook.file.FileLogStore.LOGBOOK_LOG_STORE_PATH; @@ -48,15 +49,22 @@ public class FileLogStoreTest extends TestCase { "\tat org.apache.felix.scr.impl.ComponentRegistry.checkComponentName(ComponentRegistry.java:240) ~[?:?]\n" + "\tat org.apache.felix.scr.impl.BundleComponentActivator.validateAndRegister(BundleComponentActivator.java:443) ~[?:?]"; + private final AtomicInteger lineCount = new AtomicInteger(); + @BeforeTest public void setUp() { TimeZone.setDefault(TimeZone.getTimeZone("GMT")); } + @BeforeMethod + public void reSet() { + lineCount.set(0); + } + @Test public void testParseLogJavaLine() { FileLogStore cut = new FileLogStore(); - BrooklynLogEntry brooklynLogEntry = cut.parseLogLine(JAVA_LOG_LINE); + BrooklynLogEntry brooklynLogEntry = cut.parseLogLine(JAVA_LOG_LINE, lineCount); assertNull(brooklynLogEntry.getTaskId()); assertNull(brooklynLogEntry.getEntityIds()); assertEquals("2021-05-27T11:36:59,251", brooklynLogEntry.getTimestampString()); @@ -66,12 +74,13 @@ public class FileLogStoreTest extends TestCase { assertEquals("o.a.b.c.m.i.LocalManagementContext", brooklynLogEntry.getClazz()); assertEquals("qtp158784971-235", brooklynLogEntry.getThreadName()); assertEquals("Top-level effector invocation: restart[] on BasicApplicationImpl{id=gwpndj09r8, name=Application (gwpndj09r8)}", brooklynLogEntry.getMessage()); + assertEquals(String.valueOf(1), brooklynLogEntry.getLineId()); } @Test public void testParseLogJavaLineWithExtraSpace() { FileLogStore cut = new FileLogStore(); - BrooklynLogEntry brooklynLogEntry = cut.parseLogLine(JAVA_LOG_LINE_WITH_EXTRA_SPACE); + BrooklynLogEntry brooklynLogEntry = cut.parseLogLine(JAVA_LOG_LINE_WITH_EXTRA_SPACE, lineCount); assertNull(brooklynLogEntry.getTaskId()); assertNull(brooklynLogEntry.getEntityIds()); assertEquals("2021-06-07T14:58:58,487", brooklynLogEntry.getTimestampString()); @@ -81,12 +90,13 @@ public class FileLogStoreTest extends TestCase { assertEquals("o.o.p.l.s.s.EventAdminConfigurationNotifier", brooklynLogEntry.getClazz()); assertEquals("s4j.pax.logging)", brooklynLogEntry.getThreadName()); assertEquals("Sending Event Admin notification (configuration successful) to org/ops4j/pax/logging/Configuration", brooklynLogEntry.getMessage()); + assertEquals(String.valueOf(1), brooklynLogEntry.getLineId()); } @Test public void testParseLogTaskLine() { FileLogStore cut = new FileLogStore(); - BrooklynLogEntry brooklynLogEntry = cut.parseLogLine(TASK_LOG_LINE); + BrooklynLogEntry brooklynLogEntry = cut.parseLogLine(TASK_LOG_LINE, lineCount); assertEquals("2021-05-27T11:36:59,258", brooklynLogEntry.getTimestampString()); assertEquals("Thu May 27 11:36:59 GMT 2021", brooklynLogEntry.getDatetime().toString()); assertEquals("OGObOWJs", brooklynLogEntry.getTaskId()); @@ -96,12 +106,13 @@ public class FileLogStoreTest extends TestCase { assertEquals("o.a.b.c.m.i.EffectorUtils", brooklynLogEntry.getClazz()); assertEquals("ager-WgxriwjB-43", brooklynLogEntry.getThreadName()); assertEquals("Invoking effector restart on BasicApplicationImpl{id=gwpndj09r8, name=Application (gwpndj09r8)}", brooklynLogEntry.getMessage()); + assertEquals(String.valueOf(1), brooklynLogEntry.getLineId()); } @Test public void testParseMultiLineLog() { FileLogStore cut = new FileLogStore(); - BrooklynLogEntry brooklynLogEntry = cut.parseLogLine(JAVA_LOG_MULTI_LINE_TEXT); + BrooklynLogEntry brooklynLogEntry = cut.parseLogLine(JAVA_LOG_MULTI_LINE_TEXT, lineCount); assertNull(brooklynLogEntry.getTaskId()); assertNull(brooklynLogEntry.getEntityIds()); assertEquals("2021-07-05T12:38:09,351", brooklynLogEntry.getTimestampString()); @@ -114,21 +125,22 @@ public class FileLogStoreTest extends TestCase { "org.osgi.service.component.ComponentException: The component name 'org.apache.brooklyn.ui.external.module' has already been registered by Bundle 293 (org.apache.brooklyn.ui.modularity.brooklyn-ui-external-modules) as Component of Class org.apache.brooklyn.ui.modularity.ExternalUiModule\n" + "\tat org.apache.felix.scr.impl.ComponentRegistry.checkComponentName(ComponentRegistry.java:240) ~[?:?]\n" + "\tat org.apache.felix.scr.impl.BundleComponentActivator.validateAndRegister(BundleComponentActivator.java:443) ~[?:?]", brooklynLogEntry.getMessage()); + assertEquals(String.valueOf(1), brooklynLogEntry.getLineId()); } @Test public void testParseLogWithNoDateTime() { FileLogStore cut = new FileLogStore(); - BrooklynLogEntry brooklynLogEntry = cut.parseLogLine(JAVA_LOG_LINE_WITH_NO_DATETIME); + BrooklynLogEntry brooklynLogEntry = cut.parseLogLine(JAVA_LOG_LINE_WITH_NO_DATETIME, lineCount); assertNull(brooklynLogEntry); -} + } @Test public void testParseLogWithDateTimeFormatMismatch() { ManagementContextInternal mgmt = LocalManagementContextForTests.newInstance(); mgmt.getBrooklynProperties().put(LOGBOOK_LOG_STORE_DATEFORMAT.getName(), UNEXPECTED_DATE_TIME_FORMAT); FileLogStore cut = new FileLogStore(mgmt); - BrooklynLogEntry brooklynLogEntry = cut.parseLogLine(JAVA_LOG_LINE); + BrooklynLogEntry brooklynLogEntry = cut.parseLogLine(JAVA_LOG_LINE, lineCount); assertNull(brooklynLogEntry.getTaskId()); assertNull(brooklynLogEntry.getEntityIds()); assertNull(brooklynLogEntry.getDatetime()); @@ -138,6 +150,7 @@ public class FileLogStoreTest extends TestCase { assertEquals("o.a.b.c.m.i.LocalManagementContext", brooklynLogEntry.getClazz()); assertEquals("qtp158784971-235", brooklynLogEntry.getThreadName()); assertEquals("Top-level effector invocation: restart[] on BasicApplicationImpl{id=gwpndj09r8, name=Application (gwpndj09r8)}", brooklynLogEntry.getMessage()); + assertEquals(String.valueOf(1), brooklynLogEntry.getLineId()); } @Test @@ -164,6 +177,7 @@ public class FileLogStoreTest extends TestCase { assertEquals("o.a.b.c.m.i.LocalManagementContext", firstBrooklynLogEntry.getClazz()); assertEquals("qtp158784971-235", firstBrooklynLogEntry.getThreadName()); assertEquals("Top-level effector invocation: restart[] on BasicApplicationImpl{id=gwpndj09r8, name=Application (gwpndj09r8)}", firstBrooklynLogEntry.getMessage()); + assertEquals(String.valueOf(1), firstBrooklynLogEntry.getLineId()); // Check second log line. NOTE, this is the same multiline example. BrooklynLogEntry secondBrooklynLogEntry = brooklynLogEntries.get(1); @@ -175,6 +189,7 @@ public class FileLogStoreTest extends TestCase { assertEquals("293", secondBrooklynLogEntry.getBundleId()); assertEquals("o.a.b.u.m.ExternalUiModule", secondBrooklynLogEntry.getClazz()); assertEquals("tures-3-thread-1", secondBrooklynLogEntry.getThreadName()); + assertEquals(String.valueOf(2), secondBrooklynLogEntry.getLineId()); // TODO: this log message is expected to be a multi-line one. Fix log-store RegEx to support this. // The second assertion below is the expected one, not the following one:
