This is an automated email from the ASF dual-hosted git repository.
krisztiankasa pushed a commit to branch branch-2.7
in repository https://gitbox.apache.org/repos/asf/ambari.git
The following commit(s) were added to refs/heads/branch-2.7 by this push:
new ec2072e AMBARI-24847 - Logsearch: Cannot search for a term that
includes spaces or dashes (#2790)
ec2072e is described below
commit ec2072e9e905aaa060452300b68c6a5f272b7d7b
Author: kasakrisz <33458261+kasakr...@users.noreply.github.com>
AuthorDate: Tue Jan 29 13:43:40 2019 +0100
AMBARI-24847 - Logsearch: Cannot search for a term that includes spaces or
dashes (#2790)
---
ambari-logsearch/ambari-logsearch-it/pom.xml | 10 +
.../org/apache/ambari/logsearch/solr/Solr.java | 115 +++++++
.../ambari/logsearch/solr/SolrDocumentMatcher.java | 60 ++++
.../solr/search/SearchServiceLogsTest.java | 376 +++++++++++++++++++++
.../main/configsets/audit_logs/conf/managed-schema | 2 +-
.../configsets/hadoop_logs/conf/managed-schema | 2 +-
.../AbstractDateRangeFacetQueryConverter.java | 10 +-
.../AbstractOperationHolderConverter.java | 72 ++--
.../app/services/logs-filtering-utils.service.ts | 2 +-
9 files changed, 598 insertions(+), 51 deletions(-)
diff --git a/ambari-logsearch/ambari-logsearch-it/pom.xml
b/ambari-logsearch/ambari-logsearch-it/pom.xml
index ebeb725..7f496210 100644
--- a/ambari-logsearch/ambari-logsearch-it/pom.xml
+++ b/ambari-logsearch/ambari-logsearch-it/pom.xml
@@ -60,6 +60,16 @@
<version>${solr.version}</version>
</dependency>
<dependency>
+ <groupId>org.apache.solr</groupId>
+ <artifactId>solr-core</artifactId>
+ <version>${solr.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.solr</groupId>
+ <artifactId>solr-dataimporthandler</artifactId>
+ <version>${solr.version}</version>
+ </dependency>
+ <dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
</dependency>
diff --git
a/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/solr/Solr.java
b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/solr/Solr.java
new file mode 100644
index 0000000..ee0bc8c
--- /dev/null
+++
b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/solr/Solr.java
@@ -0,0 +1,115 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.logsearch.solr;
+
+import static
org.apache.ambari.logsearch.solr.SolrConstants.AuditLogConstants.AUDIT_EVTTIME;
+import static
org.apache.ambari.logsearch.solr.SolrConstants.CommonLogConstants.ID;
+import static
org.apache.ambari.logsearch.solr.SolrConstants.CommonLogConstants.LOG_MESSAGE;
+import static
org.apache.ambari.logsearch.solr.SolrConstants.ServiceLogConstants.LEVEL;
+import static
org.apache.ambari.logsearch.solr.SolrConstants.ServiceLogConstants.LOGTIME;
+
+import java.io.File;
+import java.io.IOException;
+import java.time.OffsetDateTime;
+import java.time.format.DateTimeFormatter;
+import java.util.Date;
+import java.util.List;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.solr.client.solrj.SolrQuery;
+import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.embedded.EmbeddedSolrServer;
+import org.apache.solr.client.solrj.request.CoreAdminRequest;
+import org.apache.solr.client.solrj.response.PivotField;
+import org.apache.solr.client.solrj.response.QueryResponse;
+import org.apache.solr.common.SolrDocumentList;
+import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.core.NodeConfig;
+import org.apache.solr.core.SolrResourceLoader;
+
+public class Solr {
+ public static final DateTimeFormatter SOLR_DATETIME_FORMATTER =
DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSSX");
+
+ public static Solr core(String coreName) throws IOException,
SolrServerException {
+ assert coreName != null;
+
+ String targetLocation =
Solr.class.getProtectionDomain().getCodeSource().getLocation().getFile() + "..";
+ String logsearchConfigSetDir = targetLocation +
"/../../ambari-logsearch-server/src/main/configsets";
+ File targetConfigSetDir = new File(targetLocation + "/configsets");
+ if (targetConfigSetDir.exists())
+ FileUtils.deleteDirectory(targetConfigSetDir);
+ FileUtils.copyDirectory(new File(logsearchConfigSetDir),
targetConfigSetDir);
+ String solrHome = targetLocation + "/solr";
+ File solrHomeDir = new File(solrHome);
+ if (solrHomeDir.exists())
+ FileUtils.deleteDirectory(solrHomeDir);
+ solrHomeDir.mkdirs();
+
+ SolrResourceLoader solrResourceLoader = new
SolrResourceLoader(solrHomeDir.toPath());
+
+ NodeConfig config = new
NodeConfig.NodeConfigBuilder("embeddedSolrServerNode", solrResourceLoader)
+ .setConfigSetBaseDirectory(targetConfigSetDir.getAbsolutePath())
+ .build();
+
+ EmbeddedSolrServer embeddedSolrServer = new EmbeddedSolrServer(config,
coreName);
+
+ CoreAdminRequest.Create createRequest = new CoreAdminRequest.Create();
+ createRequest.setCoreName(coreName);
+ createRequest.setConfigSet(coreName);
+ embeddedSolrServer.request(createRequest);
+
+ return new Solr(embeddedSolrServer);
+ }
+
+ private final EmbeddedSolrServer server;
+
+ private Solr(EmbeddedSolrServer solrServer) {
+ server = solrServer;
+ }
+
+ public void addDoc(String id, String logMessage) throws SolrServerException,
IOException {
+ SolrInputDocument doc = new SolrInputDocument();
+ doc.addField(ID, id);
+ doc.addField(LOGTIME, new
Date(OffsetDateTime.now().toInstant().toEpochMilli()));
+ doc.addField(AUDIT_EVTTIME, new
Date(OffsetDateTime.now().toInstant().toEpochMilli()));
+ doc.addField(LOG_MESSAGE, logMessage);
+ doc.addField(LEVEL, "INFO");
+ server.add(doc);
+ server.commit();
+ }
+
+ public SolrDocumentList executeQuery(SolrQuery solrQuery) throws
SolrServerException, IOException {
+ return server.query(solrQuery).getResults();
+ }
+
+ public NamedList<List<PivotField>> executeFacetQuery(SolrQuery solrQuery)
throws SolrServerException, IOException {
+ QueryResponse qResp = server.query(solrQuery);
+ return qResp.getFacetPivot();
+ }
+
+ public void clear() throws Exception {
+ server.deleteByQuery("*:*");
+ server.commit();
+ }
+
+ public void close() throws IOException {
+ server.close();
+ }
+}
diff --git
a/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/solr/SolrDocumentMatcher.java
b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/solr/SolrDocumentMatcher.java
new file mode 100644
index 0000000..540e117
--- /dev/null
+++
b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/solr/SolrDocumentMatcher.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.logsearch.solr;
+
+import org.apache.solr.common.SolrDocument;
+import org.hamcrest.Description;
+import org.hamcrest.TypeSafeDiagnosingMatcher;
+
+public class SolrDocumentMatcher extends
TypeSafeDiagnosingMatcher<SolrDocument> {
+
+ public static SolrDocumentMatcher solrDoc(String expectedId) {
+ return new SolrDocumentMatcher(expectedId);
+ }
+
+ private final String expectedId;
+
+ private SolrDocumentMatcher(String expectedId) {
+ this.expectedId = expectedId;
+ }
+
+ @Override
+ protected boolean matchesSafely(SolrDocument item, Description
mismatchDescription) {
+ if (!item.containsKey("id")) {
+ mismatchDescription.appendText("SolrDocument without 'id' field.");
+ return false;
+ }
+ if (!expectedId.equals(item.get("id"))) {
+ describe(mismatchDescription, item.get("id"));
+ return false;
+ }
+ return true;
+ }
+
+ @Override
+ public void describeTo(Description description) {
+ describe(description, expectedId);
+ }
+
+ private void describe(Description description, Object id) {
+ description.appendText("SolrDocument[id=");
+ description.appendValue(id);
+ description.appendText("]");
+ }
+}
diff --git
a/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/solr/search/SearchServiceLogsTest.java
b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/solr/search/SearchServiceLogsTest.java
new file mode 100644
index 0000000..b7766c9
--- /dev/null
+++
b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/solr/search/SearchServiceLogsTest.java
@@ -0,0 +1,376 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.logsearch.solr.search;
+
+import static org.apache.ambari.logsearch.solr.Solr.SOLR_DATETIME_FORMATTER;
+import static org.apache.ambari.logsearch.solr.SolrDocumentMatcher.solrDoc;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.hasItem;
+import static org.hamcrest.Matchers.hasSize;
+import static org.hamcrest.core.Is.is;
+
+import java.io.IOException;
+import java.time.OffsetDateTime;
+import java.time.ZoneId;
+import java.util.ArrayList;
+import java.util.List;
+
+import
org.apache.ambari.logsearch.converter.BaseServiceLogRequestQueryConverter;
+import
org.apache.ambari.logsearch.converter.ServiceLogLevelDateRangeRequestQueryConverter;
+import org.apache.ambari.logsearch.model.request.impl.BaseServiceLogRequest;
+import org.apache.ambari.logsearch.model.request.impl.ServiceGraphRequest;
+import
org.apache.ambari.logsearch.model.request.impl.query.BaseServiceLogQueryRequest;
+import
org.apache.ambari.logsearch.model.request.impl.query.ServiceGraphQueryRequest;
+import org.apache.ambari.logsearch.solr.Solr;
+import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.response.PivotField;
+import org.apache.solr.common.SolrDocument;
+import org.apache.solr.common.SolrDocumentList;
+import org.apache.solr.common.util.NamedList;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.springframework.data.solr.core.DefaultQueryParser;
+import org.springframework.data.solr.core.query.SimpleQuery;
+
+public class SearchServiceLogsTest {
+ private static Solr solr;
+
+ private final BaseServiceLogRequestQueryConverter requestQueryConverter =
new BaseServiceLogRequestQueryConverter();
+
+ @BeforeClass
+ public static void setUp() throws Exception {
+ solr = Solr.core("hadoop_logs");
+ }
+
+ @AfterClass
+ public static void stopSolr() throws Exception {
+ solr.close();
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ solr.clear();
+ }
+
+ @Test
+ public void testIncludeLogMessageFilter() throws Exception {
+ solr.addDoc("0", "Unhandled exception");
+ solr.addDoc("1", "exception occurred");
+ solr.addDoc("2", "exception");
+ solr.addDoc("3", "Unhandled error");
+ solr.addDoc("4", "Error occurred");
+
+ BaseServiceLogRequest request = new BaseServiceLogQueryRequest();
+ request.setIncludeQuery("[{\"log_message\":\"exception\"}]");
+ ArrayList<SolrDocument> found = executeQuery(request);
+
+ assertThat(found, hasSize(3));
+ assertThat(found, hasItem(solrDoc("0")));
+ assertThat(found, hasItem(solrDoc("1")));
+ assertThat(found, hasItem(solrDoc("2")));
+ }
+
+ @Test
+ public void testIncludeBlankLogMessageFilter() throws Exception {
+ solr.addDoc("0", "");
+ solr.addDoc("1", "exception occurred");
+
+ BaseServiceLogRequest request = new BaseServiceLogQueryRequest();
+ request.setIncludeQuery("[{\"log_message\":\"\\\"\\\"\"}]");
+ ArrayList<SolrDocument> found = executeQuery(request);
+
+ assertThat(found, hasSize(0));
+ }
+
+ @Test
+ public void testIncludeQuotedSpaceLogMessageFilter() throws Exception {
+ solr.addDoc("0", " ");
+ solr.addDoc("1", "exception occurred");
+
+ BaseServiceLogRequest request = new BaseServiceLogQueryRequest();
+ request.setIncludeQuery("[{\"log_message\":\"\\\" \\\"\"}]");
+ ArrayList<SolrDocument> found = executeQuery(request);
+
+ assertThat(found, hasSize(0));
+ }
+
+ @Test
+ public void testIncludeSpaceLogMessageFilter() throws Exception {
+ solr.addDoc("0", " ");
+ solr.addDoc("1", "exception occurred");
+
+ BaseServiceLogRequest request = new BaseServiceLogQueryRequest();
+ request.setIncludeQuery("[{\"log_message\":\" \"}]");
+ ArrayList<SolrDocument> found = executeQuery(request);
+
+ assertThat(found, hasSize(2));
+ }
+
+ private SolrDocumentList executeQuery(BaseServiceLogRequest request) throws
IOException, SolrServerException {
+ SimpleQuery simpleQuery = requestQueryConverter.convert(request);
+ assert simpleQuery != null;
+ return solr.executeQuery(new
DefaultQueryParser().doConstructSolrQuery(simpleQuery));
+ }
+
+
+ @Test
+ public void testIncludeMultipleLogMessageFilterContainsWildcard() throws
Exception {
+ solr.addDoc("0", "Unhandled exception");
+ solr.addDoc("1", "exception occurred");
+ solr.addDoc("2", "exception");
+ solr.addDoc("3", "Unhandled error");
+ solr.addDoc("4", "Error occurred");
+
+ BaseServiceLogRequest request = new BaseServiceLogQueryRequest();
+
request.setIncludeQuery("[{\"log_message\":\"exceptio*\"},{\"log_message\":\"unha*\"}]");
+ SolrDocumentList found = executeQuery(request);
+
+ assertThat(found, hasSize(1));
+ assertThat(found, hasItem(solrDoc("0")));
+ }
+
+ @Test
+ public void testIncludeMultipleTerms() throws Exception {
+ solr.addDoc("0", "At line 112 an unhandled exception occurred when");
+ solr.addDoc("1", "exception occurred");
+ solr.addDoc("2", "exception");
+ solr.addDoc("3", "Unhandled error");
+ solr.addDoc("4", "Error occurred");
+
+ BaseServiceLogRequest request = new BaseServiceLogQueryRequest();
+ request.setIncludeQuery("[{\"log_message\":\"\\\"Unhandled
exception\\\"\"}]");
+ SolrDocumentList found = executeQuery(request);
+
+ assertThat(found, hasSize(1));
+ assertThat(found, hasItem(solrDoc("0")));
+ }
+
+ @Test
+ public void testIncludeMultipleTermsDelimitedByDash() throws Exception {
+ solr.addDoc("0", "Unhandled exception");
+ solr.addDoc("1", "At line 112 an unhandled-exception occurred when");
+ solr.addDoc("2", "exception occurred");
+ solr.addDoc("3", "exception");
+ solr.addDoc("4", "Unhandled error");
+ solr.addDoc("5", "Error occurred");
+
+ BaseServiceLogRequest request = new BaseServiceLogQueryRequest();
+
request.setIncludeQuery("[{\"log_message\":\"\\\"Unhandled-exception\\\"\"}]");
+ SolrDocumentList found = executeQuery(request);
+
+ assertThat(found, hasSize(1));
+ assertThat(found, hasItem(solrDoc("1")));
+ }
+
+ @Test
+ public void testIncludeMultipleTermsDelimitedByPlusSign() throws Exception {
+ solr.addDoc("0", "Unhandled exception");
+ solr.addDoc("1", "Unhandled+exception");
+ solr.addDoc("2", "Unhandled-exception");
+ solr.addDoc("3", "exception occurred");
+ solr.addDoc("4", "exception");
+ solr.addDoc("5", "Unhandled error");
+ solr.addDoc("6", "Error occurred");
+
+ BaseServiceLogRequest request = new BaseServiceLogQueryRequest();
+
request.setIncludeQuery("[{\"log_message\":\"\\\"Unhandled+exception\\\"\"}]");
+ SolrDocumentList found = executeQuery(request);
+
+ assertThat(found, hasSize(1));
+ assertThat(found, hasItem(solrDoc("1")));
+ }
+
+ @Test
+ public void testIncludeALongMessage() throws Exception {
+ solr.addDoc("6", "[ ]
org.apache.solr.cloud.autoscaling.OverseerTriggerThread
(OverseerTriggerThread.java:400) - Error in trigger 'index_size_trigger'
configuration, trigger config ignored: {\\r\\n
\\\"aboveBytes\\\":53687091200,\\r\\n
\\\"aboveOp\\\":\\\"SPLITSHARD\\\",\\r\\n
\\\"event\\\":\\\"indexSize\\\",\\r\\n \\\"waitFor\\\":1,\\r\\n
\\\"actions\\\":[\\r\\n {\\r\\n
\\\"name\\\":\\\"compute_plan\\\",\\r\\n
\\\"class\\\":\\\"solr.ComputePlanAction\\\"},\\r\\n [...]
+ solr.addDoc("7", "[ ]
org.apache.solr.cloud.autoscaling.OverseerTriggerThread
(OverseerTriggerThread.java:422) - Something else");
+ solr.addDoc("8", "[ ] org.apache.solr.cloud.OverseerTriggerThread
(OverseerTriggerThread.java:400) - Different package");
+
+ BaseServiceLogRequest request = new BaseServiceLogQueryRequest();
+
request.setIncludeQuery("[{\"log_message\":\"\\\"org.apache.solr.cloud.autoscaling.OverseerTriggerThread
(OverseerTriggerThread.java:400)\\\"\"}]");
+ SolrDocumentList found = executeQuery(request);
+
+ assertThat(found, hasSize(1));
+ assertThat(found, hasItem(solrDoc("6")));
+ }
+
+ @Test
+ public void testIncludeMultipleWhitespaceDelimitedWords() throws Exception {
+ solr.addDoc("0", "At line 112 an unhandled exception occurred when");
+ solr.addDoc("1", "At line 112 an unhandled exception occurred when");
+ solr.addDoc("2", "At line 112 an unhandled \texception occurred when");
+ solr.addDoc("3", "At line 112 an exception occurred when");
+
+ BaseServiceLogRequest request = new BaseServiceLogQueryRequest();
+ request.setIncludeQuery("[{\"log_message\":\"\\\"Unhandled
exception\\\"\"}]");
+ SolrDocumentList found = executeQuery(request);
+
+ assertThat(found, hasSize(3));
+ assertThat(found, hasItem(solrDoc("0")));
+ assertThat(found, hasItem(solrDoc("1")));
+ assertThat(found, hasItem(solrDoc("2")));
+ }
+
+ @Test
+ public void testIncludeEMailAddress() throws Exception {
+ solr.addDoc("0", "Email address: j...@hortonworks.com");
+ solr.addDoc("1", "Another document");
+
+ BaseServiceLogRequest request = new BaseServiceLogQueryRequest();
+ request.setIncludeQuery("[{\"log_message\":\"j...@hortonworks.com\"}]");
+ SolrDocumentList found = executeQuery(request);
+
+ assertThat(found, hasSize(1));
+ assertThat(found, hasItem(solrDoc("0")));
+ }
+
+
+ private final ServiceLogLevelDateRangeRequestQueryConverter
dateRangeRequestQueryConverter = new
ServiceLogLevelDateRangeRequestQueryConverter();
+
+ @Test
+ public void testDateRangeRequestIncludeEMailAddress() throws Exception {
+ solr.addDoc("0", "Email address: j...@hortonworks.com");
+ solr.addDoc("1", "Another document d...@hortonworks.com");
+
+ ServiceGraphRequest request =
serviceGraphRequest("[{\"log_message\":\"j...@hortonworks.com\"}]");
+ NamedList<List<PivotField>> found = executeQuery(request);
+
+ assertThat(found.size(), is(1));
+ assertThat(found.get("level").get(0).getCount(), is(1));
+ }
+
+ private ServiceGraphRequest serviceGraphRequest(String includeQuery) {
+ ServiceGraphRequest request = new ServiceGraphQueryRequest();
+
request.setFrom(SOLR_DATETIME_FORMATTER.format(OffsetDateTime.now(ZoneId.of("UTC")).minusDays(1)));
+
request.setTo(SOLR_DATETIME_FORMATTER.format(OffsetDateTime.now(ZoneId.of("UTC")).plusDays(1)));
+ request.setIncludeQuery(includeQuery);
+ return request;
+ }
+
+ private NamedList<List<PivotField>> executeQuery(ServiceGraphRequest
request) throws IOException, SolrServerException {
+ return
solr.executeFacetQuery(dateRangeRequestQueryConverter.convert(request));
+ }
+
+ @Test
+ public void testDateRangeRequestIncludeLogMessageFilter() throws Exception {
+ solr.addDoc("0", "Unhandled exception");
+ solr.addDoc("1", "exception occurred");
+ solr.addDoc("2", "exception");
+ solr.addDoc("3", "Unhandled error");
+ solr.addDoc("4", "Error occurred");
+
+ ServiceGraphRequest request =
serviceGraphRequest("[{\"log_message\":\"exception\"}]");
+ NamedList<List<PivotField>> found = executeQuery(request);
+
+ assertThat(found.size(), is(1));
+ assertThat(found.get("level").get(0).getCount(), is(3));
+ }
+
+ @Test
+ public void
testDateRangeRequestIncludeMultipleLogMessageFilterContainsWildcard() throws
Exception {
+ solr.addDoc("0", "Unhandled exception");
+ solr.addDoc("1", "exception occurred");
+ solr.addDoc("2", "exception");
+ solr.addDoc("3", "Unhandled error");
+ solr.addDoc("4", "Error occurred");
+
+ ServiceGraphRequest request =
serviceGraphRequest("[{\"log_message\":\"exceptio*\"},{\"log_message\":\"unha*\"}]");
+ NamedList<List<PivotField>> found = executeQuery(request);
+
+ assertThat(found.size(), is(1));
+ assertThat(found.get("level").get(0).getCount(), is(1));
+ }
+
+ @Test
+ public void testDateRangeRequestIncludeMultipleTerms() throws Exception {
+ solr.addDoc("0", "At line 112 an unhandled exception occurred when");
+ solr.addDoc("1", "exception occurred");
+ solr.addDoc("2", "exception");
+ solr.addDoc("3", "Unhandled error");
+ solr.addDoc("4", "Error occurred");
+
+ ServiceGraphRequest request =
serviceGraphRequest("[{\"log_message\":\"\\\"Unhandled exception\\\"\"}]");
+ NamedList<List<PivotField>> found = executeQuery(request);
+
+ assertThat(found.size(), is(1));
+ assertThat(found.get("level").get(0).getCount(), is(1));
+ }
+
+ @Test
+ public void testDateRangeRequestIncludeMultipleTermsDelimitedByDash() throws
Exception {
+ solr.addDoc("0", "Unhandled exception");
+ solr.addDoc("1", "At line 112 an unhandled-exception occurred when");
+ solr.addDoc("2", "exception occurred");
+ solr.addDoc("3", "exception");
+ solr.addDoc("4", "Unhandled error");
+ solr.addDoc("5", "Error occurred");
+
+ ServiceGraphRequest request =
serviceGraphRequest("[{\"log_message\":\"\\\"Unhandled-exception\\\"\"}]");
+ NamedList<List<PivotField>> found = executeQuery(request);
+
+ assertThat(found.size(), is(1));
+ assertThat(found.get("level").get(0).getCount(), is(1));
+ }
+
+ @Test
+ public void testDateRangeRequestIncludeMultipleTermsDelimitedByPlusSign()
throws Exception {
+ solr.addDoc("0", "Unhandled exception");
+ solr.addDoc("1", "Unhandled+exception");
+ solr.addDoc("2", "Unhandled-exception");
+ solr.addDoc("3", "exception occurred");
+ solr.addDoc("4", "exception");
+ solr.addDoc("5", "Unhandled error");
+ solr.addDoc("6", "Error occurred");
+
+ ServiceGraphRequest request =
serviceGraphRequest("[{\"log_message\":\"\\\"Unhandled+exception\\\"\"}]");
+ NamedList<List<PivotField>> found = executeQuery(request);
+
+ assertThat(found.size(), is(1));
+ assertThat(found.get("level").get(0).getCount(), is(1));
+ }
+
+ @Test
+ public void testDateRangeRequestIncludeALongMessage() throws Exception {
+ solr.addDoc("6", "[ ]
org.apache.solr.cloud.autoscaling.OverseerTriggerThread
(OverseerTriggerThread.java:400) - Error in trigger 'index_size_trigger'
configuration, trigger config ignored: {\\r\\n
\\\"aboveBytes\\\":53687091200,\\r\\n
\\\"aboveOp\\\":\\\"SPLITSHARD\\\",\\r\\n
\\\"event\\\":\\\"indexSize\\\",\\r\\n \\\"waitFor\\\":1,\\r\\n
\\\"actions\\\":[\\r\\n {\\r\\n
\\\"name\\\":\\\"compute_plan\\\",\\r\\n
\\\"class\\\":\\\"solr.ComputePlanAction\\\"},\\r\\n [...]
+ solr.addDoc("7", "[ ]
org.apache.solr.cloud.autoscaling.OverseerTriggerThread
(OverseerTriggerThread.java:422) - Something else");
+ solr.addDoc("8", "[ ] org.apache.solr.cloud.OverseerTriggerThread
(OverseerTriggerThread.java:400) - Different package");
+
+ ServiceGraphRequest request =
serviceGraphRequest("[{\"log_message\":\"\\\"org.apache.solr.cloud.autoscaling.OverseerTriggerThread
(OverseerTriggerThread.java:400)\\\"\"}]");
+ NamedList<List<PivotField>> found = executeQuery(request);
+
+ assertThat(found.size(), is(1));
+ assertThat(found.get("level").get(0).getCount(), is(1));
+ }
+
+ @Test
+ public void testDateRangeRequestIncludeMultipleWhitespaceDelimitedWords()
throws Exception {
+ solr.addDoc("0", "At line 112 an unhandled exception occurred when");
+ solr.addDoc("1", "At line 112 an unhandled exception occurred when");
+ solr.addDoc("2", "At line 112 an unhandled \texception occurred when");
+ solr.addDoc("3", "At line 112 an exception occurred when");
+
+ ServiceGraphRequest request =
serviceGraphRequest("[{\"log_message\":\"\\\"Unhandled exception\\\"\"}]");
+ NamedList<List<PivotField>> found = executeQuery(request);
+
+ assertThat(found.size(), is(1));
+ assertThat(found.get("level").get(0).getCount(), is(3));
+ }
+}
diff --git
a/ambari-logsearch/ambari-logsearch-server/src/main/configsets/audit_logs/conf/managed-schema
b/ambari-logsearch/ambari-logsearch-server/src/main/configsets/audit_logs/conf/managed-schema
index fd90ada..6896918 100644
---
a/ambari-logsearch/ambari-logsearch-server/src/main/configsets/audit_logs/conf/managed-schema
+++
b/ambari-logsearch/ambari-logsearch-server/src/main/configsets/audit_logs/conf/managed-schema
@@ -91,7 +91,7 @@
<field name="ip" type="key_lower_case"/>
<field name="level" type="key_lower_case"/>
<field name="logType" type="key_lower_case" multiValued="false"/>
- <field name="log_message" type="key_lower_case" multiValued="false"
omitNorms="false"/>
+ <field name="log_message" type="text_ws" multiValued="false"
omitNorms="false"/>
<field name="logfile_line_number" type="tint" omitNorms="false"/>
<field name="logger_name" type="key_lower_case"/>
<field name="message_md5" type="string" multiValued="false"/>
diff --git
a/ambari-logsearch/ambari-logsearch-server/src/main/configsets/hadoop_logs/conf/managed-schema
b/ambari-logsearch/ambari-logsearch-server/src/main/configsets/hadoop_logs/conf/managed-schema
index b1fda99..168427e 100644
---
a/ambari-logsearch/ambari-logsearch-server/src/main/configsets/hadoop_logs/conf/managed-schema
+++
b/ambari-logsearch/ambari-logsearch-server/src/main/configsets/hadoop_logs/conf/managed-schema
@@ -96,7 +96,7 @@
<field name="ip" type="string" multiValued="false"/>
<field name="level" type="lowercase" multiValued="false"/>
<field name="line_number" type="tint" omitNorms="false"/>
- <field name="log_message" type="text_general" multiValued="false"
omitNorms="false"/>
+ <field name="log_message" type="text_ws" multiValued="false"
omitNorms="false"/>
<field name="logfile_line_number" type="tint" omitNorms="false"/>
<field name="logger_name" type="string" multiValued="false"/>
<field name="logtime" type="tdate" multiValued="false" docValues="true"/>
diff --git
a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/converter/AbstractDateRangeFacetQueryConverter.java
b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/converter/AbstractDateRangeFacetQueryConverter.java
index 2143f56..02a5872 100644
---
a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/converter/AbstractDateRangeFacetQueryConverter.java
+++
b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/converter/AbstractDateRangeFacetQueryConverter.java
@@ -18,14 +18,14 @@
*/
package org.apache.ambari.logsearch.converter;
+import java.util.Locale;
+
import org.apache.ambari.logsearch.common.LogSearchConstants;
import org.apache.ambari.logsearch.model.request.DateRangeParamDefinition;
import org.apache.ambari.logsearch.model.request.UnitParamDefinition;
import org.apache.commons.lang.StringUtils;
import org.apache.solr.client.solrj.SolrQuery;
-import java.util.Locale;
-
public abstract class AbstractDateRangeFacetQueryConverter<SOURCE extends
DateRangeParamDefinition & UnitParamDefinition>
extends AbstractOperationHolderConverter<SOURCE , SolrQuery> {
@@ -40,9 +40,9 @@ public abstract class
AbstractDateRangeFacetQueryConverter<SOURCE extends DateRa
solrQuery.setFacetLimit(-1);
solrQuery.setFacetSort(LogSearchConstants.FACET_INDEX);
solrQuery.add("facet.range", "{!tag=r1}" + getDateFieldName());
- solrQuery.add(String.format(Locale.ROOT, "f.%s.%s", new
Object[]{getDateFieldName(), "facet.range.start"}), request.getFrom());
- solrQuery.add(String.format(Locale.ROOT, "f.%s.%s", new
Object[]{getDateFieldName(), "facet.range.end"}), request.getTo());
- solrQuery.add(String.format(Locale.ROOT, "f.%s.%s", new
Object[]{getDateFieldName(), "facet.range.gap"}), unit);
+ solrQuery.add(String.format(Locale.ROOT, "f.%s.%s", getDateFieldName(),
"facet.range.start"), request.getFrom());
+ solrQuery.add(String.format(Locale.ROOT, "f.%s.%s", getDateFieldName(),
"facet.range.end"), request.getTo());
+ solrQuery.add(String.format(Locale.ROOT, "f.%s.%s", getDateFieldName(),
"facet.range.gap"), unit);
solrQuery.remove("sort");
solrQuery.setRows(0);
solrQuery.setStart(0);
diff --git
a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/converter/AbstractOperationHolderConverter.java
b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/converter/AbstractOperationHolderConverter.java
index 5e7f440..54ae154 100644
---
a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/converter/AbstractOperationHolderConverter.java
+++
b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/converter/AbstractOperationHolderConverter.java
@@ -18,28 +18,30 @@
*/
package org.apache.ambari.logsearch.converter;
-import com.google.common.base.Splitter;
-import com.google.gson.Gson;
-import com.google.gson.reflect.TypeToken;
+import static java.util.Collections.singletonList;
+import static
org.apache.ambari.logsearch.solr.SolrConstants.ServiceLogConstants.LOG_MESSAGE;
+import static org.apache.commons.lang3.StringUtils.isNotBlank;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import javax.inject.Inject;
+
import org.apache.ambari.logsearch.common.LogType;
import org.apache.ambari.logsearch.dao.SolrSchemaFieldDao;
import org.apache.ambari.logsearch.util.SolrUtil;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang.StringUtils;
-import org.apache.commons.lang.text.StrTokenizer;
import org.apache.solr.client.solrj.SolrQuery;
import org.springframework.data.solr.core.query.Criteria;
import org.springframework.data.solr.core.query.Query;
import org.springframework.data.solr.core.query.SimpleFilterQuery;
import org.springframework.data.solr.core.query.SimpleStringCriteria;
-import javax.inject.Inject;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import static
org.apache.ambari.logsearch.solr.SolrConstants.ServiceLogConstants.LOG_MESSAGE;
+import com.google.common.base.Splitter;
+import com.google.gson.Gson;
+import com.google.gson.reflect.TypeToken;
public abstract class AbstractOperationHolderConverter <REQUEST_TYPE,
QUERY_TYPE>
extends AbstractConverterAware<REQUEST_TYPE, QUERY_TYPE> {
@@ -79,7 +81,7 @@ public abstract class AbstractOperationHolderConverter
<REQUEST_TYPE, QUERY_TYPE
public Query addInFiltersIfNotNullAndEnabled(Query query, String value,
String field, boolean condition) {
if (value != null && condition) {
- List<String> values = value.length() == 0 ? Arrays.asList("-1") :
splitValueAsList(value, ",");
+ List<String> values = value.length() == 0 ? singletonList("-1") :
splitValueAsList(value, ",");
addInFilterQuery(query, field, values);
}
return query;
@@ -87,7 +89,7 @@ public abstract class AbstractOperationHolderConverter
<REQUEST_TYPE, QUERY_TYPE
public SolrQuery addInFiltersIfNotNullAndEnabled(SolrQuery query, String
value, String field, boolean condition) {
if (condition) {
- List<String> valuesList = value.length() == 0 ? Arrays.asList("\\-1") :
splitValueAsList(value, ",");
+ List<String> valuesList = value.length() == 0 ? singletonList("\\-1") :
splitValueAsList(value, ",");
if (valuesList.size() > 1) {
query.addFilterQuery(String.format("%s:(%s)", field,
StringUtils.join(valuesList, " OR ")));
} else {
@@ -198,41 +200,25 @@ public abstract class AbstractOperationHolderConverter
<REQUEST_TYPE, QUERY_TYPE
public abstract LogType getLogType();
private void addLogMessageFilter(Query query, String value, boolean negate) {
- StrTokenizer tokenizer = new StrTokenizer(value, ' ', '"');
- for (String token : tokenizer.getTokenArray()) {
- token = token.trim();
- if (token.contains(" ") || !token.startsWith("*") &&
!token.endsWith("*")) {
- addFilterQuery(query, new
Criteria(LOG_MESSAGE).is(SolrUtil.escapeQueryChars(token)), negate);
- } else if (token.startsWith("*") && token.endsWith("*")) {
- String plainToken = StringUtils.substring(token, 1, -1);
- addFilterQuery(query, new
Criteria(LOG_MESSAGE).contains(SolrUtil.escapeQueryChars(plainToken)), negate);
- } else if (token.startsWith("*") && !token.endsWith("*")) {
- String plainToken = StringUtils.substring(token, 1);
- addFilterQuery(query, new
Criteria(LOG_MESSAGE).endsWith(SolrUtil.escapeQueryChars(plainToken)), negate);
- } else if (!token.startsWith("*") && token.endsWith("*")) {
- String plainToken = StringUtils.substring(token, 0, -1);
- addFilterQuery(query, new
Criteria(LOG_MESSAGE).startsWith(SolrUtil.escapeQueryChars(plainToken)),
negate);
- }
+ value = value.trim();
+ if (StringUtils.startsWith(value, "\"") &&
StringUtils.endsWith(value,"\"")) {
+ value = String.format("\"%s\"",
SolrUtil.escapeQueryChars(StringUtils.substring(value, 1, -1)));
+ addFilterQuery(query, new Criteria(LOG_MESSAGE).expression(value),
negate);
+ }
+ else if (isNotBlank(value)){
+ addFilterQuery(query, new
Criteria(LOG_MESSAGE).expression(SolrUtil.escapeQueryChars(value)), negate);
}
}
private void addLogMessageFilter(SolrQuery query, String value, boolean
negate) {
- StrTokenizer tokenizer = new StrTokenizer(value, ' ', '"');
String negateToken = negate ? "-" : "";
- for (String token : tokenizer.getTokenArray()) {
- token = token.trim();
- if (token.contains(" ") || !token.startsWith("*") &&
!token.endsWith("*")) {
- query.addFilterQuery(String.format("%s%s:%s", negateToken,
LOG_MESSAGE, SolrUtil.escapeQueryChars(token)));
- } else if (token.startsWith("*") && token.endsWith("*")) {
- String plainToken = StringUtils.substring(token, 1, -1);
- query.addFilterQuery(String.format("%s%s:%s", negateToken,
LOG_MESSAGE, SolrUtil.escapeQueryChars(plainToken)));
- } else if (token.startsWith("*") && !token.endsWith("*")) {
- String plainToken = StringUtils.substring(token, 1);
- query.addFilterQuery(String.format("%s%s:%s", negateToken,
LOG_MESSAGE, SolrUtil.escapeQueryChars(plainToken)));
- } else if (!token.startsWith("*") && token.endsWith("*")) {
- String plainToken = StringUtils.substring(token, 0, -1);
- query.addFilterQuery(String.format("%s%s:%s", negateToken,
LOG_MESSAGE, SolrUtil.escapeQueryChars(plainToken)));
- }
+ value = value.trim();
+ if (StringUtils.startsWith(value, "\"") &&
StringUtils.endsWith(value,"\"")) {
+ value = String.format("\"%s\"",
SolrUtil.escapeQueryChars(StringUtils.substring(value, 1, -1)));
+ query.addFilterQuery(String.format("%s%s:%s", negateToken, LOG_MESSAGE,
value));
+ }
+ else {
+ query.addFilterQuery(String.format("%s%s:%s", negateToken, LOG_MESSAGE,
SolrUtil.escapeQueryChars(value)));
}
}
diff --git
a/ambari-logsearch/ambari-logsearch-web/src/app/services/logs-filtering-utils.service.ts
b/ambari-logsearch/ambari-logsearch-web/src/app/services/logs-filtering-utils.service.ts
index 89f8a6d..9fddade 100644
---
a/ambari-logsearch/ambari-logsearch-web/src/app/services/logs-filtering-utils.service.ts
+++
b/ambari-logsearch/ambari-logsearch-web/src/app/services/logs-filtering-utils.service.ts
@@ -355,7 +355,7 @@ export class LogsFilteringUtilsService {
return item.isExclude === isExclude;
}).map((parameter: SearchBoxParameter): HomogeneousObject<string> => {
return {
- [parameter.name]: parameter.value.replace(/\s/g, '+')
+ [parameter.name]: parameter.value
};
});
}
