This is an automated email from the ASF dual-hosted git repository.
zqr10159 pushed a commit to branch 2.0.0
in repository https://gitbox.apache.org/repos/asf/hertzbeat.git
The following commit(s) were added to refs/heads/2.0.0 by this push:
new 499130500b Support log list IN filters
499130500b is described below
commit 499130500b64567e9d662453919d54fbb80f1313
Author: Logic <[email protected]>
AuthorDate: Tue Jun 9 20:56:40 2026 +0800
Support log list IN filters
---
.../logs/service/impl/LogQueryServiceImpl.java | 185 ++++++++++++++++++++-
.../logs/controller/LogQueryControllerTest.java | 63 +++++++
2 files changed, 241 insertions(+), 7 deletions(-)
diff --git
a/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/logs/service/impl/LogQueryServiceImpl.java
b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/logs/service/impl/LogQueryServiceImpl.java
index 66369ae671..315782d62d 100644
---
a/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/logs/service/impl/LogQueryServiceImpl.java
+++
b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/logs/service/impl/LogQueryServiceImpl.java
@@ -21,6 +21,7 @@ import java.time.Instant;
import java.time.LocalDateTime;
import java.time.ZoneId;
import java.time.format.DateTimeFormatter;
+import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
@@ -31,6 +32,8 @@ import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import org.apache.hertzbeat.common.entity.manager.EntityIdentity;
@@ -67,6 +70,12 @@ public class LogQueryServiceImpl implements LogQueryService {
private static final int MAX_CONTEXT_LIMIT = 50;
private static final long DEFAULT_CONTEXT_WINDOW_MS = 300_000L;
private static final String LOG_FILTER_NEGATION_PREFIX = "!";
+ private static final String LOG_FILTER_IN_PREFIX = "__hz_in__:";
+ private static final String LOG_FILTER_NOT_IN_PREFIX = "__hz_not_in__:";
+ private static final String LOG_FILTER_VALUE_DELIMITER = "\u001F";
+ private static final Pattern LOG_FILTER_LIST_OPERATOR_PATTERN =
Pattern.compile(
+ "^\\s*([A-Za-z0-9._:-]+)\\s+(NOT\\s+IN|IN)\\s*(\\(.+\\))\\s*$",
+ Pattern.CASE_INSENSITIVE);
private static final Set<String> WORKSPACE_INFRA_SERVICE_NAMES = Set.of(
"otelcol-contrib",
@@ -128,8 +137,8 @@ public class LogQueryServiceImpl implements LogQueryService
{
String serviceName, String serviceNamespace,
String environment,
String resourceFilter, String attributeFilter,
Integer pageIndex, Integer pageSize, boolean
hideInternal, boolean hideNoise) {
- Map<String, String> resourceFilters =
parseLogAttributeFilter(resourceFilter);
- Map<String, String> attributeFilters =
parseLogAttributeFilter(attributeFilter);
+ Map<String, String> resourceFilters =
parseLogAttributeFilter(resourceFilter, true);
+ Map<String, String> attributeFilters =
parseLogAttributeFilter(attributeFilter, true);
return getPagedLogs(start, end, traceId, spanId, severityNumber,
severityText, search,
serviceName, serviceNamespace, environment, resourceFilters,
attributeFilters,
pageIndex, pageSize, hideInternal, hideNoise);
@@ -143,8 +152,8 @@ public class LogQueryServiceImpl implements LogQueryService
{
Integer pageIndex, Integer pageSize, boolean
hideInternal, boolean hideNoise) {
LogServiceContext context =
resolveEntityFirstLogServiceContext(entityId, serviceName, serviceNamespace,
environment);
Map<String, String> resourceFilters = removeEntityScopeResourceFilters(
- context, parseLogAttributeFilter(resourceFilter));
- Map<String, String> attributeFilters =
parseLogAttributeFilter(attributeFilter);
+ context, parseLogAttributeFilter(resourceFilter, true));
+ Map<String, String> attributeFilters =
parseLogAttributeFilter(attributeFilter, true);
return getPagedLogs(start, end, traceId, spanId, severityNumber,
severityText, search,
context.serviceName(), context.serviceNamespace(),
context.environment(), resourceFilters, attributeFilters,
pageIndex, pageSize, hideInternal, hideNoise);
@@ -831,6 +840,12 @@ public class LogQueryServiceImpl implements
LogQueryService {
Sort sort = Sort.by(Sort.Direction.DESC, "timeUnixNano");
PageRequest pageRequest = PageRequest.of(resolvedPageIndex,
resolvedPageSize, sort);
+ if (hasComplexAttributeFilters(resourceFilters, attributeFilters)) {
+ return getRowFilteredPagedLogs(start, end, traceId, spanId,
severityNumber, severityText, search,
+ serviceName, serviceNamespace, environment,
resourceFilters, attributeFilters,
+ pageRequest, offset, resolvedPageSize, hideInternal,
hideNoise);
+ }
+
if (hasWorkspaceContext()) {
return getWorkspacePagedLogs(start, end, traceId, spanId,
severityNumber, severityText, search,
serviceName, serviceNamespace, environment,
resourceFilters, attributeFilters,
@@ -1169,16 +1184,30 @@ public class LogQueryServiceImpl implements
LogQueryService {
|| (attributeFilters != null && !attributeFilters.isEmpty());
}
+ private boolean hasComplexAttributeFilters(Map<String, String>
resourceFilters, Map<String, String> attributeFilters) {
+ return hasComplexAttributeFilterValues(resourceFilters) ||
hasComplexAttributeFilterValues(attributeFilters);
+ }
+
+ private boolean hasComplexAttributeFilterValues(Map<String, String>
filters) {
+ return filters != null &&
filters.values().stream().anyMatch(this::isListLogAttributeFilter);
+ }
+
private Map<String, String> parseLogAttributeFilter(String
filterExpression) {
+ return parseLogAttributeFilter(filterExpression, false);
+ }
+
+ private Map<String, String> parseLogAttributeFilter(String
filterExpression, boolean allowListOperators) {
if (!StringUtils.hasText(filterExpression)) {
return Collections.emptyMap();
}
Map<String, String> filters = new HashMap<>();
- String[] tokens =
filterExpression.trim().split("(?i)\\s+and\\s+|\\s*,\\s*");
- for (String token : tokens) {
+ for (String token : splitLogFilterClauses(filterExpression)) {
if (!StringUtils.hasText(token)) {
continue;
}
+ if (allowListOperators && appendLogFilterListValues(filters,
token)) {
+ continue;
+ }
boolean negate = false;
int separatorIndex = token.indexOf("!=");
if (separatorIndex >= 0) {
@@ -1202,6 +1231,115 @@ public class LogQueryServiceImpl implements
LogQueryService {
return filters.isEmpty() ? Collections.emptyMap() :
Map.copyOf(filters);
}
+ private boolean appendLogFilterListValues(Map<String, String> filters,
String token) {
+ Matcher matcher = LOG_FILTER_LIST_OPERATOR_PATTERN.matcher(token);
+ if (!matcher.matches()) {
+ return false;
+ }
+ String key = matcher.group(1).trim();
+ String operator = matcher.group(2).trim().replaceAll("\\s+", " ");
+ String valueList = matcher.group(3).trim();
+ if (!isSafeAttributeKey(key) || valueList.length() < 2
+ || !valueList.startsWith("(") || !valueList.endsWith(")")) {
+ return false;
+ }
+ List<String> values = splitLogFilterListValues(valueList.substring(1,
valueList.length() - 1)).stream()
+ .map(value -> stripFilterQuotes(value.trim()))
+ .filter(StringUtils::hasText)
+ .distinct()
+ .toList();
+ if (values.isEmpty()) {
+ return false;
+ }
+ String prefix = "not in".equalsIgnoreCase(operator) ?
LOG_FILTER_NOT_IN_PREFIX : LOG_FILTER_IN_PREFIX;
+ filters.put(key, prefix + String.join(LOG_FILTER_VALUE_DELIMITER,
values));
+ return true;
+ }
+
+ private List<String> splitLogFilterClauses(String filterExpression) {
+ List<String> clauses = new ArrayList<>();
+ StringBuilder current = new StringBuilder();
+ int depth = 0;
+ char quote = 0;
+ for (int index = 0; index < filterExpression.length(); index++) {
+ char character = filterExpression.charAt(index);
+ if (quote != 0) {
+ current.append(character);
+ if (character == quote) {
+ quote = 0;
+ }
+ continue;
+ }
+ if (character == '\'' || character == '"') {
+ quote = character;
+ current.append(character);
+ continue;
+ }
+ if (character == '(') {
+ depth++;
+ current.append(character);
+ continue;
+ }
+ if (character == ')') {
+ depth = Math.max(0, depth - 1);
+ current.append(character);
+ continue;
+ }
+ if (depth == 0 && character == ',') {
+ addLogFilterClause(clauses, current);
+ continue;
+ }
+ if (depth == 0 && isLogFilterAndDelimiter(filterExpression,
index)) {
+ addLogFilterClause(clauses, current);
+ index += 4;
+ continue;
+ }
+ current.append(character);
+ }
+ addLogFilterClause(clauses, current);
+ return clauses;
+ }
+
+ private List<String> splitLogFilterListValues(String values) {
+ List<String> result = new ArrayList<>();
+ StringBuilder current = new StringBuilder();
+ char quote = 0;
+ for (int index = 0; index < values.length(); index++) {
+ char character = values.charAt(index);
+ if (quote != 0) {
+ current.append(character);
+ if (character == quote) {
+ quote = 0;
+ }
+ continue;
+ }
+ if (character == '\'' || character == '"') {
+ quote = character;
+ current.append(character);
+ continue;
+ }
+ if (character == ',') {
+ addLogFilterClause(result, current);
+ continue;
+ }
+ current.append(character);
+ }
+ addLogFilterClause(result, current);
+ return result;
+ }
+
+ private void addLogFilterClause(List<String> clauses, StringBuilder
current) {
+ String clause = current.toString().trim();
+ if (StringUtils.hasText(clause)) {
+ clauses.add(clause);
+ }
+ current.setLength(0);
+ }
+
+ private boolean isLogFilterAndDelimiter(String value, int index) {
+ return index + 5 <= value.length() && value.regionMatches(true, index,
" and ", 0, 5);
+ }
+
private Map<String, String>
removeEntityScopeResourceFilters(LogServiceContext context,
Map<String,
String> resourceFilters) {
if (context == null || resourceFilters == null ||
resourceFilters.isEmpty()) {
@@ -1280,13 +1418,21 @@ public class LogQueryServiceImpl implements
LogQueryService {
return true;
}
if (source == null || source.isEmpty()) {
- return
expectedAttributes.values().stream().allMatch(this::isNegatedLogAttributeFilter);
+ return
expectedAttributes.values().stream().allMatch(this::isExclusionLogAttributeFilter);
}
return expectedAttributes.entrySet().stream()
.allMatch(entry ->
matchesAttributeFilter(resolveMapValue(source, entry.getKey()),
entry.getValue()));
}
private boolean matchesAttributeFilter(String actualValue, String
expectedValue) {
+ if (isInLogAttributeFilter(expectedValue)) {
+ return
splitListLogAttributeValues(expectedValue.substring(LOG_FILTER_IN_PREFIX.length())).stream()
+ .anyMatch(expected ->
matchesOptionalResourceValue(actualValue, expected));
+ }
+ if (isNotInLogAttributeFilter(expectedValue)) {
+ return
splitListLogAttributeValues(expectedValue.substring(LOG_FILTER_NOT_IN_PREFIX.length())).stream()
+ .noneMatch(expected ->
matchesOptionalResourceValue(actualValue, expected));
+ }
if (isNegatedLogAttributeFilter(expectedValue)) {
return !matchesOptionalResourceValue(actualValue,
expectedValue.substring(LOG_FILTER_NEGATION_PREFIX.length()));
}
@@ -1297,6 +1443,31 @@ public class LogQueryServiceImpl implements
LogQueryService {
return expectedValue != null &&
expectedValue.startsWith(LOG_FILTER_NEGATION_PREFIX);
}
+ private boolean isExclusionLogAttributeFilter(String expectedValue) {
+ return isNegatedLogAttributeFilter(expectedValue) ||
isNotInLogAttributeFilter(expectedValue);
+ }
+
+ private boolean isListLogAttributeFilter(String expectedValue) {
+ return isInLogAttributeFilter(expectedValue) ||
isNotInLogAttributeFilter(expectedValue);
+ }
+
+ private boolean isInLogAttributeFilter(String expectedValue) {
+ return expectedValue != null &&
expectedValue.startsWith(LOG_FILTER_IN_PREFIX);
+ }
+
+ private boolean isNotInLogAttributeFilter(String expectedValue) {
+ return expectedValue != null &&
expectedValue.startsWith(LOG_FILTER_NOT_IN_PREFIX);
+ }
+
+ private List<String> splitListLogAttributeValues(String encodedValues) {
+ if (!StringUtils.hasText(encodedValues)) {
+ return List.of();
+ }
+ return
List.of(encodedValues.split(Pattern.quote(LOG_FILTER_VALUE_DELIMITER),
-1)).stream()
+ .filter(StringUtils::hasText)
+ .toList();
+ }
+
private boolean shouldHideWorkspaceLog(LogEntry logEntry, boolean
hideInternal, boolean hideNoise) {
String serviceName = resolveServiceName(logEntry);
if (!StringUtils.hasText(serviceName)) {
diff --git
a/hertzbeat-observability/src/test/java/org/apache/hertzbeat/observability/logs/controller/LogQueryControllerTest.java
b/hertzbeat-observability/src/test/java/org/apache/hertzbeat/observability/logs/controller/LogQueryControllerTest.java
index cede67c0f2..8cf11ff05e 100644
---
a/hertzbeat-observability/src/test/java/org/apache/hertzbeat/observability/logs/controller/LogQueryControllerTest.java
+++
b/hertzbeat-observability/src/test/java/org/apache/hertzbeat/observability/logs/controller/LogQueryControllerTest.java
@@ -719,6 +719,69 @@ class LogQueryControllerTest {
eq(Map.of("service.version", "!1.2.3")),
eq(Map.of("http.route", "!/checkout")));
}
+ @Test
+ void testListLogsAppliesInAndNotInFiltersWithRowFallback() throws
Exception {
+ LogEntry stableLog = LogEntry.builder()
+ .timeUnixNano(1734005477630000000L)
+ .severityText("INFO")
+ .body("stable checkout log")
+ .resource(new HashMap<>(java.util.Map.of(
+ "service.name", "checkout",
+ "service.version", "1.2.3",
+ "host.name", "checkout-1")))
+ .attributes(new HashMap<>(java.util.Map.of("http.route",
"/checkout")))
+ .build();
+ LogEntry canaryLog = LogEntry.builder()
+ .timeUnixNano(1734005477640000000L)
+ .severityText("INFO")
+ .body("canary checkout log")
+ .resource(new HashMap<>(java.util.Map.of(
+ "service.name", "checkout",
+ "service.version", "1.2.3",
+ "host.name", "checkout-canary")))
+ .attributes(new HashMap<>(java.util.Map.of("http.route",
"/checkout")))
+ .build();
+ LogEntry cartLog = LogEntry.builder()
+ .timeUnixNano(1734005477650000000L)
+ .severityText("INFO")
+ .body("cart checkout log")
+ .resource(new HashMap<>(java.util.Map.of(
+ "service.name", "checkout",
+ "service.version", "1.2.4",
+ "host.name", "checkout-2")))
+ .attributes(new HashMap<>(java.util.Map.of("http.route",
"/cart")))
+ .build();
+ LogEntry otherVersionLog = LogEntry.builder()
+ .timeUnixNano(1734005477660000000L)
+ .severityText("INFO")
+ .body("other checkout log")
+ .resource(new HashMap<>(java.util.Map.of(
+ "service.name", "checkout",
+ "service.version", "2.0.0",
+ "host.name", "checkout-3")))
+ .attributes(new HashMap<>(java.util.Map.of("http.route",
"/checkout")))
+ .build();
+ when(historyDataReader.queryLogsByMultipleConditions(any(), any(),
any(),
+ any(), any(), any(), any())).thenReturn(List.of(stableLog,
canaryLog, cartLog, otherVersionLog));
+
+ mockMvc.perform(MockMvcRequestBuilders.get("/api/logs/list")
+ .param("resourceFilter", "service.version IN
(\"1.2.3\", '1.2.4') "
+ + "and host.name NOT IN ('checkout-canary')")
+ .param("attributeFilter", "http.route IN
('/checkout')"))
+ .andExpect(status().isOk())
+ .andExpect(jsonPath("$.code").value((int)
CommonConstants.SUCCESS_CODE))
+ .andExpect(jsonPath("$.data.content.length()").value(1))
+ .andExpect(jsonPath("$.data.content[0].body").value("stable
checkout log"))
+ .andExpect(jsonPath("$.data.totalElements").value(1));
+
+ verify(historyDataReader).queryLogsByMultipleConditions(any(), any(),
any(),
+ any(), any(), any(), any());
+ verify(historyDataReader,
never()).countLogsByMultipleConditions(any(), any(), any(),
+ any(), any(), any(), any());
+ verify(historyDataReader,
never()).queryLogsByMultipleConditionsWithPagination(any(), any(),
+ any(), any(), any(), any(), any(), anyInt(), anyInt());
+ }
+
@Test
void testContextLogsReturnsSelectedLogWithBoundedBeforeAndAfterRows()
throws Exception {
long selectedTime = 1734005477630000000L;
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
