This is an automated email from the ASF dual-hosted git repository.
snoopdave pushed a commit to branch roller-6.1.x
in repository https://gitbox.apache.org/repos/asf/roller.git
The following commit(s) were added to refs/heads/roller-6.1.x by this push:
new 666e793fe Limit search queries to the search fields and stop indexing
comment email addresses (#225)
666e793fe is described below
commit 666e793fed12f01a578a2175349db4c6339ac446
Author: David M. Johnson <[email protected]>
AuthorDate: Thu Oct 29 22:46:12 2026 -0400
Limit search queries to the search fields and stop indexing comment email
addresses (#225)
* Limit search queries to the search fields and stop indexing comment email
addresses
* Note the search field change in CHANGES.md
* Accept *:* in search again as a match-all query
* Check search fields on the parsed query and keep only search data in the
index
---
CHANGES.md | 6 ++
.../business/search/lucene/FieldConstants.java | 3 +
.../business/search/lucene/IndexOperation.java | 28 ------
.../business/search/lucene/LuceneIndexManager.java | 23 +++++
.../business/search/lucene/SearchOperation.java | 49 +++++++++-
.../business/search/IndexManagerTest.java | 10 ++
.../business/search/lucene/IndexedFieldsTest.java | 108 +++++++++++++++++++++
.../search/lucene/SearchFieldsQueryParserTest.java | 69 +++++++++++++
8 files changed, 267 insertions(+), 29 deletions(-)
diff --git a/CHANGES.md b/CHANGES.md
index 819373e77..13306195d 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -82,6 +82,12 @@
administrator has disabled is now refused whenever its OpenID URL is used to
sign in, on every sign-in form, as user name and password sign-in already
does.
+- **Search queries can name only the search fields.** A weblog search may
+ name entry content, title or comment text (for example `title:roller`). A
+ query that names another field is treated like a query that cannot be
+ parsed. Author user names and comment author names and email addresses are
+ no longer added to the search index. An index built by an earlier version
+ is rebuilt in the background at the first startup after the upgrade.
### Bug fixes
diff --git
a/app/src/main/java/org/apache/roller/weblogger/business/search/lucene/FieldConstants.java
b/app/src/main/java/org/apache/roller/weblogger/business/search/lucene/FieldConstants.java
index bfbd9a2db..0ad8f3bf0 100644
---
a/app/src/main/java/org/apache/roller/weblogger/business/search/lucene/FieldConstants.java
+++
b/app/src/main/java/org/apache/roller/weblogger/business/search/lucene/FieldConstants.java
@@ -43,4 +43,7 @@ public final class FieldConstants {
public static final String CONSTANT_V = "v";
public static final String WEBSITE_HANDLE = "handle";
public static final String LOCALE = "locale";
+
+ /** Fields that earlier versions indexed; an index holding one is rebuilt.
*/
+ static final java.util.List<String> REMOVED_FIELDS =
java.util.List.of(USERNAME, C_EMAIL, C_NAME);
}
diff --git
a/app/src/main/java/org/apache/roller/weblogger/business/search/lucene/IndexOperation.java
b/app/src/main/java/org/apache/roller/weblogger/business/search/lucene/IndexOperation.java
index 7c251e8bf..d4357af0c 100644
---
a/app/src/main/java/org/apache/roller/weblogger/business/search/lucene/IndexOperation.java
+++
b/app/src/main/java/org/apache/roller/weblogger/business/search/lucene/IndexOperation.java
@@ -75,31 +75,17 @@ public abstract class IndexOperation implements Runnable {
"search.index.comments", true);
String commentContent = "";
- String commentEmail = "";
- String commentName = "";
if (indexComments) {
List<WeblogEntryComment> comments = data.getComments();
if (comments != null) {
- StringBuilder commentEmailBld = new StringBuilder();
StringBuilder commentContentBld = new StringBuilder();
- StringBuilder commentNameBld = new StringBuilder();
for (WeblogEntryComment comment : comments) {
if (comment.getContent() != null) {
commentContentBld.append(comment.getContent());
commentContentBld.append(",");
}
- if (comment.getEmail() != null) {
- commentEmailBld.append(comment.getEmail());
- commentEmailBld.append(",");
- }
- if (comment.getName() != null) {
- commentNameBld.append(comment.getName());
- commentNameBld.append(",");
- }
}
- commentEmail = commentEmailBld.toString();
commentContent = commentContentBld.toString();
- commentName = commentNameBld.toString();
}
}
@@ -113,12 +99,6 @@ public abstract class IndexOperation implements Runnable {
doc.add(new StringField(FieldConstants.WEBSITE_HANDLE, data
.getWebsite().getHandle(), Field.Store.YES));
- // text, don't index deleted/disabled users of a group blog
- if (data.getCreator() != null) {
- doc.add(new TextField(FieldConstants.USERNAME, data.getCreator()
- .getUserName().toLowerCase(), Field.Store.YES));
- }
-
// text
doc.add(new TextField(FieldConstants.TITLE, data.getTitle(),
Field.Store.YES));
@@ -152,14 +132,6 @@ public abstract class IndexOperation implements Runnable {
doc.add(new TextField(FieldConstants.C_CONTENT, commentContent,
Field.Store.NO));
- // keyword
- doc.add(new StringField(FieldConstants.C_EMAIL, commentEmail,
- Field.Store.YES));
-
- // keyword
- doc.add(new StringField(FieldConstants.C_NAME, commentName,
- Field.Store.YES));
-
return doc;
}
diff --git
a/app/src/main/java/org/apache/roller/weblogger/business/search/lucene/LuceneIndexManager.java
b/app/src/main/java/org/apache/roller/weblogger/business/search/lucene/LuceneIndexManager.java
index 7a7e5b512..a2e734a16 100644
---
a/app/src/main/java/org/apache/roller/weblogger/business/search/lucene/LuceneIndexManager.java
+++
b/app/src/main/java/org/apache/roller/weblogger/business/search/lucene/LuceneIndexManager.java
@@ -39,6 +39,7 @@ import
org.apache.lucene.analysis.miscellaneous.LimitTokenCountAnalyzer;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.FieldInfos;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
@@ -151,6 +152,14 @@ public class LuceneIndexManager implements IndexManager {
try {
synchronized(this) {
reader = DirectoryReader.open(getIndexDirectory());
+ if (hasRemovedFields(reader)) {
+ logger.info("Search index holds fields that are no
longer indexed, scheduling rebuild.");
+ reader.close();
+ reader = null;
+ inconsistentAtStartup = true;
+ deleteIndex();
+ createIndex(getIndexDirectory());
+ }
}
} catch (IOException | IllegalArgumentException ex) { // IAE
for incompatible codecs
logger.warn("Failed to open search index, scheduling
rebuild.", ex);
@@ -343,6 +352,20 @@ public class LuceneIndexManager implements IndexManager {
return null;
}
+ /**
+ * True if the index holds a field that earlier versions indexed but this
+ * one does not, so the index must be rebuilt to drop that data.
+ */
+ static boolean hasRemovedFields(IndexReader reader) {
+ FieldInfos fields = FieldInfos.getMergedFieldInfos(reader);
+ for (String removed : FieldConstants.REMOVED_FIELDS) {
+ if (fields.fieldInfo(removed) != null) {
+ return true;
+ }
+ }
+ return false;
+ }
+
private boolean indexExists() {
try {
return DirectoryReader.indexExists(getIndexDirectory());
diff --git
a/app/src/main/java/org/apache/roller/weblogger/business/search/lucene/SearchOperation.java
b/app/src/main/java/org/apache/roller/weblogger/business/search/lucene/SearchOperation.java
index a1f2f4e1e..04b1305bf 100644
---
a/app/src/main/java/org/apache/roller/weblogger/business/search/lucene/SearchOperation.java
+++
b/app/src/main/java/org/apache/roller/weblogger/business/search/lucene/SearchOperation.java
@@ -19,9 +19,12 @@
package org.apache.roller.weblogger.business.search.lucene;
import java.io.IOException;
+import java.util.HashSet;
+import java.util.Set;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.Term;
import org.apache.lucene.queryparser.classic.MultiFieldQueryParser;
@@ -30,6 +33,7 @@ import org.apache.lucene.search.BooleanClause;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
+import org.apache.lucene.search.QueryVisitor;
import org.apache.lucene.search.Sort;
import org.apache.lucene.search.SortField;
import org.apache.lucene.search.TermQuery;
@@ -104,7 +108,7 @@ public class SearchOperation extends ReadFromIndexOperation
{
IndexReader reader = manager.getSharedIndexReader();
searcher = new IndexSearcher(reader);
- MultiFieldQueryParser multiParser = new MultiFieldQueryParser(
+ MultiFieldQueryParser multiParser = new SearchFieldsQueryParser(
SEARCH_FIELDS, LuceneIndexManager.getAnalyzer());
// Make it an AND by default. Comment this out for an or (default)
@@ -229,4 +233,47 @@ public class SearchOperation extends
ReadFromIndexOperation {
this.locale = locale;
}
+
+ /**
+ * Query parser that accepts explicit field names only for the fields that
+ * searches are meant to cover. Other index fields hold data that is not
+ * shown on the weblog, so a query naming one is a parse error. The check
+ * walks the parsed query, so it covers every kind of query the parser
+ * can build.
+ */
+ static class SearchFieldsQueryParser extends MultiFieldQueryParser {
+
+ private final Set<String> searchFields;
+
+ SearchFieldsQueryParser(String[] fields, Analyzer analyzer) {
+ super(fields, analyzer);
+ this.searchFields = Set.of(fields);
+ }
+
+ @Override
+ public Query parse(String query) throws ParseException {
+ Query parsed = super.parse(query);
+ Set<String> fields = new HashSet<>();
+ parsed.visit(new QueryVisitor() {
+ @Override
+ public boolean acceptField(String field) {
+ fields.add(field);
+ return true;
+ }
+
+ @Override
+ public void consumeTerms(Query q, Term... terms) {
+ for (Term t : terms) {
+ fields.add(t.field());
+ }
+ }
+ });
+ for (String field : fields) {
+ if (!searchFields.contains(field)) {
+ throw new ParseException("Cannot search field: " + field);
+ }
+ }
+ return parsed;
+ }
+ }
}
diff --git
a/app/src/test/java/org/apache/roller/weblogger/business/search/IndexManagerTest.java
b/app/src/test/java/org/apache/roller/weblogger/business/search/IndexManagerTest.java
index 7f146b0b9..2c98d3be5 100644
---
a/app/src/test/java/org/apache/roller/weblogger/business/search/IndexManagerTest.java
+++
b/app/src/test/java/org/apache/roller/weblogger/business/search/IndexManagerTest.java
@@ -25,7 +25,9 @@ import org.apache.commons.logging.LogFactory;
import org.apache.roller.util.RollerConstants;
import org.apache.roller.weblogger.TestUtils;
import org.apache.roller.weblogger.business.WeblogEntryManager;
+import org.apache.roller.weblogger.WebloggerException;
import org.apache.roller.weblogger.business.WebloggerFactory;
+import org.apache.roller.weblogger.business.search.lucene.FieldConstants;
import org.apache.roller.weblogger.pojos.User;
import org.apache.roller.weblogger.pojos.Weblog;
import org.apache.roller.weblogger.pojos.WeblogCategory;
@@ -37,6 +39,7 @@ import org.junit.jupiter.api.Test;
import static
org.apache.roller.weblogger.ui.rendering.model.SearchResultsModel.RESULTS_PER_PAGE;
import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
/**
* Test Search Manager business layer operations.
@@ -97,6 +100,13 @@ public class IndexManagerTest {
WebloggerFactory.getWeblogger().getUrlStrategy());
assertEquals(1, result.getResults().size());
+ // index fields that are not search fields cannot be queried
+ String handlePrefix = testWeblog.getHandle().substring(0,
3).toLowerCase() + "*";
+ assertThrows(WebloggerException.class, () -> indexManager.search(
+ FieldConstants.WEBSITE_HANDLE + ":" + handlePrefix,
+ testWeblog.getHandle(), null, testWeblog.getLocale(), 0,
RESULTS_PER_PAGE,
+ WebloggerFactory.getWeblogger().getUrlStrategy()));
+
} finally {
for (WeblogEntry entry : entries) {
indexManager.removeEntryIndexOperation(TestUtils.getManagedWeblogEntry(entry));
diff --git
a/app/src/test/java/org/apache/roller/weblogger/business/search/lucene/IndexedFieldsTest.java
b/app/src/test/java/org/apache/roller/weblogger/business/search/lucene/IndexedFieldsTest.java
new file mode 100644
index 000000000..802977b2c
--- /dev/null
+++
b/app/src/test/java/org/apache/roller/weblogger/business/search/lucene/IndexedFieldsTest.java
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.roller.weblogger.business.search.lucene;
+
+import java.sql.Timestamp;
+import java.util.List;
+
+import org.apache.lucene.analysis.standard.StandardAnalyzer;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.StringField;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.store.ByteBuffersDirectory;
+import org.apache.lucene.store.Directory;
+import org.apache.roller.weblogger.pojos.User;
+import org.apache.roller.weblogger.pojos.Weblog;
+import org.apache.roller.weblogger.pojos.WeblogEntry;
+import org.apache.roller.weblogger.pojos.WeblogEntryComment;
+import org.junit.jupiter.api.Test;
+
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+/**
+ * The index holds only what search needs, and an index built by an earlier
+ * version with more fields is found so it can be rebuilt.
+ */
+class IndexedFieldsTest {
+
+ @Test
+ void entryDocumentHoldsNoAuthorOrCommenterDetails() {
+ Document doc = new IndexOperation(null) {
+ @Override
+ protected void doRun() {
+ }
+ }.getDocument(entry());
+
+ assertNotNull(doc.getField(FieldConstants.ID));
+ assertNotNull(doc.getField(FieldConstants.C_CONTENT));
+ for (String removed : FieldConstants.REMOVED_FIELDS) {
+ assertNull(doc.getField(removed), removed);
+ }
+ }
+
+ @Test
+ void indexWithARemovedFieldIsFound() throws Exception {
+ assertTrue(hasRemovedFields(FieldConstants.C_EMAIL));
+ assertFalse(hasRemovedFields(FieldConstants.TITLE));
+ }
+
+ private static boolean hasRemovedFields(String field) throws Exception {
+ try (Directory dir = new ByteBuffersDirectory()) {
+ try (IndexWriter writer = new IndexWriter(dir, new
IndexWriterConfig(new StandardAnalyzer()))) {
+ Document doc = new Document();
+ doc.add(new StringField(field, "value", Field.Store.YES));
+ writer.addDocument(doc);
+ }
+ try (DirectoryReader reader = DirectoryReader.open(dir)) {
+ return LuceneIndexManager.hasRemovedFields(reader);
+ }
+ }
+ }
+
+ private static WeblogEntry entry() {
+ WeblogEntryComment comment = mock(WeblogEntryComment.class);
+ when(comment.getContent()).thenReturn("nice post");
+ when(comment.getName()).thenReturn("Reader");
+ when(comment.getEmail()).thenReturn("[email protected]");
+
+ Weblog weblog = mock(Weblog.class);
+ when(weblog.getHandle()).thenReturn("team");
+ User author = mock(User.class);
+ when(author.getUserName()).thenReturn("author");
+
+ WeblogEntry entry = mock(WeblogEntry.class);
+ when(entry.getId()).thenReturn("id1");
+ when(entry.getWebsite()).thenReturn(weblog);
+ when(entry.getCreator()).thenReturn(author);
+ when(entry.getTitle()).thenReturn("Title");
+ when(entry.getLocale()).thenReturn("en");
+ when(entry.getText()).thenReturn("Text");
+ Timestamp now = new Timestamp(System.currentTimeMillis());
+ when(entry.getUpdateTime()).thenReturn(now);
+ when(entry.getPubTime()).thenReturn(now);
+ when(entry.getComments()).thenReturn(List.of(comment));
+ return entry;
+ }
+}
diff --git
a/app/src/test/java/org/apache/roller/weblogger/business/search/lucene/SearchFieldsQueryParserTest.java
b/app/src/test/java/org/apache/roller/weblogger/business/search/lucene/SearchFieldsQueryParserTest.java
new file mode 100644
index 000000000..047921cb5
--- /dev/null
+++
b/app/src/test/java/org/apache/roller/weblogger/business/search/lucene/SearchFieldsQueryParserTest.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.roller.weblogger.business.search.lucene;
+
+import org.apache.lucene.analysis.standard.StandardAnalyzer;
+import org.apache.lucene.queryparser.classic.ParseException;
+import org.apache.lucene.search.MatchAllDocsQuery;
+import org.junit.jupiter.api.Test;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+/**
+ * Searches may name only the fields they are meant to cover.
+ */
+class SearchFieldsQueryParserTest {
+
+ private static SearchOperation.SearchFieldsQueryParser parser() {
+ return new SearchOperation.SearchFieldsQueryParser(
+ new String[] {FieldConstants.CONTENT, FieldConstants.TITLE,
FieldConstants.C_CONTENT},
+ new StandardAnalyzer());
+ }
+
+ @Test
+ void plainAndSearchFieldQueriesParse() throws Exception {
+ assertNotNull(parser().parse("roller weblog"));
+ assertNotNull(parser().parse("roll*"));
+ assertNotNull(parser().parse("title:roller"));
+ assertNotNull(parser().parse("content:roll*"));
+ assertNotNull(parser().parse("\"apache roller\"~2"));
+ }
+
+ @Test
+ void matchAllParses() throws Exception {
+ assertEquals(new MatchAllDocsQuery(), parser().parse("*:*"));
+ }
+
+ @Test
+ void otherFieldsAreRefused() {
+ for (String query : new String[] {
+ FieldConstants.C_EMAIL + ":a*",
+ FieldConstants.C_EMAIL + ":[email protected]",
+ FieldConstants.C_NAME + ":reader",
+ FieldConstants.USERNAME + ":admin",
+ FieldConstants.ANCHOR + ":[a TO z]",
+ FieldConstants.C_EMAIL + ":/re.*/",
+ FieldConstants.C_EMAIL + ":reader~",
+ "*:a*",
+ FieldConstants.C_EMAIL + ":*",
+ "roller AND " + FieldConstants.C_EMAIL + ":r?ader"}) {
+ assertThrows(ParseException.class, () -> parser().parse(query),
query);
+ }
+ }
+}