This is an automated email from the ASF dual-hosted git repository.

snoopdave pushed a commit to branch roller-6.1.x
in repository https://gitbox.apache.org/repos/asf/roller.git


The following commit(s) were added to refs/heads/roller-6.1.x by this push:
     new 666e793fe Limit search queries to the search fields and stop indexing 
comment email addresses (#225)
666e793fe is described below

commit 666e793fed12f01a578a2175349db4c6339ac446
Author: David M. Johnson <[email protected]>
AuthorDate: Thu Oct 29 22:46:12 2026 -0400

    Limit search queries to the search fields and stop indexing comment email 
addresses (#225)
    
    * Limit search queries to the search fields and stop indexing comment email 
addresses
    * Note the search field change in CHANGES.md
    * Accept *:* in search again as a match-all query
    * Check search fields on the parsed query and keep only search data in the 
index
---
 CHANGES.md                                         |   6 ++
 .../business/search/lucene/FieldConstants.java     |   3 +
 .../business/search/lucene/IndexOperation.java     |  28 ------
 .../business/search/lucene/LuceneIndexManager.java |  23 +++++
 .../business/search/lucene/SearchOperation.java    |  49 +++++++++-
 .../business/search/IndexManagerTest.java          |  10 ++
 .../business/search/lucene/IndexedFieldsTest.java  | 108 +++++++++++++++++++++
 .../search/lucene/SearchFieldsQueryParserTest.java |  69 +++++++++++++
 8 files changed, 267 insertions(+), 29 deletions(-)

diff --git a/CHANGES.md b/CHANGES.md
index 819373e77..13306195d 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -82,6 +82,12 @@
   administrator has disabled is now refused whenever its OpenID URL is used to
   sign in, on every sign-in form, as user name and password sign-in already
   does.
+- **Search queries can name only the search fields.** A weblog search may
+  name entry content, title or comment text (for example `title:roller`). A
+  query that names another field is treated like a query that cannot be
+  parsed. Author user names and comment author names and email addresses are
+  no longer added to the search index. An index built by an earlier version
+  is rebuilt in the background at the first startup after the upgrade.
 
 ### Bug fixes
 
diff --git 
a/app/src/main/java/org/apache/roller/weblogger/business/search/lucene/FieldConstants.java
 
b/app/src/main/java/org/apache/roller/weblogger/business/search/lucene/FieldConstants.java
index bfbd9a2db..0ad8f3bf0 100644
--- 
a/app/src/main/java/org/apache/roller/weblogger/business/search/lucene/FieldConstants.java
+++ 
b/app/src/main/java/org/apache/roller/weblogger/business/search/lucene/FieldConstants.java
@@ -43,4 +43,7 @@ public final class FieldConstants {
     public static final String CONSTANT_V = "v";
     public static final String WEBSITE_HANDLE = "handle";
     public static final String LOCALE = "locale";
+
+    /** Fields that earlier versions indexed; an index holding one is rebuilt. 
*/
+    static final java.util.List<String> REMOVED_FIELDS = 
java.util.List.of(USERNAME, C_EMAIL, C_NAME);
 }
diff --git 
a/app/src/main/java/org/apache/roller/weblogger/business/search/lucene/IndexOperation.java
 
b/app/src/main/java/org/apache/roller/weblogger/business/search/lucene/IndexOperation.java
index 7c251e8bf..d4357af0c 100644
--- 
a/app/src/main/java/org/apache/roller/weblogger/business/search/lucene/IndexOperation.java
+++ 
b/app/src/main/java/org/apache/roller/weblogger/business/search/lucene/IndexOperation.java
@@ -75,31 +75,17 @@ public abstract class IndexOperation implements Runnable {
                 "search.index.comments", true);
 
         String commentContent = "";
-        String commentEmail = "";
-        String commentName = "";
         if (indexComments) {
             List<WeblogEntryComment> comments = data.getComments();
             if (comments != null) {
-                StringBuilder commentEmailBld = new StringBuilder();
                 StringBuilder commentContentBld = new StringBuilder();
-                StringBuilder commentNameBld = new StringBuilder();
                 for (WeblogEntryComment comment : comments) {
                     if (comment.getContent() != null) {
                         commentContentBld.append(comment.getContent());
                         commentContentBld.append(",");
                     }
-                    if (comment.getEmail() != null) {
-                        commentEmailBld.append(comment.getEmail());
-                        commentEmailBld.append(",");
-                    }
-                    if (comment.getName() != null) {
-                        commentNameBld.append(comment.getName());
-                        commentNameBld.append(",");
-                    }
                 }
-                commentEmail = commentEmailBld.toString();
                 commentContent = commentContentBld.toString();
-                commentName = commentNameBld.toString();
             }
         }
 
@@ -113,12 +99,6 @@ public abstract class IndexOperation implements Runnable {
         doc.add(new StringField(FieldConstants.WEBSITE_HANDLE, data
                 .getWebsite().getHandle(), Field.Store.YES));
 
-        // text, don't index deleted/disabled users of a group blog
-        if (data.getCreator() != null) {
-            doc.add(new TextField(FieldConstants.USERNAME, data.getCreator()
-                    .getUserName().toLowerCase(), Field.Store.YES));
-        }
-
         // text
         doc.add(new TextField(FieldConstants.TITLE, data.getTitle(),
                 Field.Store.YES));
@@ -152,14 +132,6 @@ public abstract class IndexOperation implements Runnable {
         doc.add(new TextField(FieldConstants.C_CONTENT, commentContent,
                 Field.Store.NO));
 
-        // keyword
-        doc.add(new StringField(FieldConstants.C_EMAIL, commentEmail,
-                Field.Store.YES));
-
-        // keyword
-        doc.add(new StringField(FieldConstants.C_NAME, commentName,
-                Field.Store.YES));
-
         return doc;
     }
 
diff --git 
a/app/src/main/java/org/apache/roller/weblogger/business/search/lucene/LuceneIndexManager.java
 
b/app/src/main/java/org/apache/roller/weblogger/business/search/lucene/LuceneIndexManager.java
index 7a7e5b512..a2e734a16 100644
--- 
a/app/src/main/java/org/apache/roller/weblogger/business/search/lucene/LuceneIndexManager.java
+++ 
b/app/src/main/java/org/apache/roller/weblogger/business/search/lucene/LuceneIndexManager.java
@@ -39,6 +39,7 @@ import 
org.apache.lucene.analysis.miscellaneous.LimitTokenCountAnalyzer;
 import org.apache.lucene.analysis.standard.StandardAnalyzer;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.FieldInfos;
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.IndexWriter;
 import org.apache.lucene.index.IndexWriterConfig;
@@ -151,6 +152,14 @@ public class LuceneIndexManager implements IndexManager {
                 try {
                     synchronized(this) {
                         reader = DirectoryReader.open(getIndexDirectory());
+                        if (hasRemovedFields(reader)) {
+                            logger.info("Search index holds fields that are no 
longer indexed, scheduling rebuild.");
+                            reader.close();
+                            reader = null;
+                            inconsistentAtStartup = true;
+                            deleteIndex();
+                            createIndex(getIndexDirectory());
+                        }
                     }
                 } catch (IOException | IllegalArgumentException ex) {  // IAE 
for incompatible codecs
                     logger.warn("Failed to open search index, scheduling 
rebuild.", ex);
@@ -343,6 +352,20 @@ public class LuceneIndexManager implements IndexManager {
         return null;
     }
 
+    /**
+     * True if the index holds a field that earlier versions indexed but this
+     * one does not, so the index must be rebuilt to drop that data.
+     */
+    static boolean hasRemovedFields(IndexReader reader) {
+        FieldInfos fields = FieldInfos.getMergedFieldInfos(reader);
+        for (String removed : FieldConstants.REMOVED_FIELDS) {
+            if (fields.fieldInfo(removed) != null) {
+                return true;
+            }
+        }
+        return false;
+    }
+
     private boolean indexExists() {
         try {
             return DirectoryReader.indexExists(getIndexDirectory());
diff --git 
a/app/src/main/java/org/apache/roller/weblogger/business/search/lucene/SearchOperation.java
 
b/app/src/main/java/org/apache/roller/weblogger/business/search/lucene/SearchOperation.java
index a1f2f4e1e..04b1305bf 100644
--- 
a/app/src/main/java/org/apache/roller/weblogger/business/search/lucene/SearchOperation.java
+++ 
b/app/src/main/java/org/apache/roller/weblogger/business/search/lucene/SearchOperation.java
@@ -19,9 +19,12 @@
 package org.apache.roller.weblogger.business.search.lucene;
 
 import java.io.IOException;
+import java.util.HashSet;
+import java.util.Set;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
+import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.queryparser.classic.MultiFieldQueryParser;
@@ -30,6 +33,7 @@ import org.apache.lucene.search.BooleanClause;
 import org.apache.lucene.search.BooleanQuery;
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.Query;
+import org.apache.lucene.search.QueryVisitor;
 import org.apache.lucene.search.Sort;
 import org.apache.lucene.search.SortField;
 import org.apache.lucene.search.TermQuery;
@@ -104,7 +108,7 @@ public class SearchOperation extends ReadFromIndexOperation 
{
             IndexReader reader = manager.getSharedIndexReader();
             searcher = new IndexSearcher(reader);
 
-            MultiFieldQueryParser multiParser = new MultiFieldQueryParser(
+            MultiFieldQueryParser multiParser = new SearchFieldsQueryParser(
                     SEARCH_FIELDS, LuceneIndexManager.getAnalyzer());
 
             // Make it an AND by default. Comment this out for an or (default)
@@ -229,4 +233,47 @@ public class SearchOperation extends 
ReadFromIndexOperation {
         this.locale = locale;
     }
 
+
+    /**
+     * Query parser that accepts explicit field names only for the fields that
+     * searches are meant to cover. Other index fields hold data that is not
+     * shown on the weblog, so a query naming one is a parse error. The check
+     * walks the parsed query, so it covers every kind of query the parser
+     * can build.
+     */
+    static class SearchFieldsQueryParser extends MultiFieldQueryParser {
+
+        private final Set<String> searchFields;
+
+        SearchFieldsQueryParser(String[] fields, Analyzer analyzer) {
+            super(fields, analyzer);
+            this.searchFields = Set.of(fields);
+        }
+
+        @Override
+        public Query parse(String query) throws ParseException {
+            Query parsed = super.parse(query);
+            Set<String> fields = new HashSet<>();
+            parsed.visit(new QueryVisitor() {
+                @Override
+                public boolean acceptField(String field) {
+                    fields.add(field);
+                    return true;
+                }
+
+                @Override
+                public void consumeTerms(Query q, Term... terms) {
+                    for (Term t : terms) {
+                        fields.add(t.field());
+                    }
+                }
+            });
+            for (String field : fields) {
+                if (!searchFields.contains(field)) {
+                    throw new ParseException("Cannot search field: " + field);
+                }
+            }
+            return parsed;
+        }
+    }
 }
diff --git 
a/app/src/test/java/org/apache/roller/weblogger/business/search/IndexManagerTest.java
 
b/app/src/test/java/org/apache/roller/weblogger/business/search/IndexManagerTest.java
index 7f146b0b9..2c98d3be5 100644
--- 
a/app/src/test/java/org/apache/roller/weblogger/business/search/IndexManagerTest.java
+++ 
b/app/src/test/java/org/apache/roller/weblogger/business/search/IndexManagerTest.java
@@ -25,7 +25,9 @@ import org.apache.commons.logging.LogFactory;
 import org.apache.roller.util.RollerConstants;
 import org.apache.roller.weblogger.TestUtils;
 import org.apache.roller.weblogger.business.WeblogEntryManager;
+import org.apache.roller.weblogger.WebloggerException;
 import org.apache.roller.weblogger.business.WebloggerFactory;
+import org.apache.roller.weblogger.business.search.lucene.FieldConstants;
 import org.apache.roller.weblogger.pojos.User;
 import org.apache.roller.weblogger.pojos.Weblog;
 import org.apache.roller.weblogger.pojos.WeblogCategory;
@@ -37,6 +39,7 @@ import org.junit.jupiter.api.Test;
 
 import static 
org.apache.roller.weblogger.ui.rendering.model.SearchResultsModel.RESULTS_PER_PAGE;
 import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
 
 /**
  * Test Search Manager business layer operations.
@@ -97,6 +100,13 @@ public class IndexManagerTest {
                 WebloggerFactory.getWeblogger().getUrlStrategy());
             assertEquals(1, result.getResults().size());
 
+            // index fields that are not search fields cannot be queried
+            String handlePrefix = testWeblog.getHandle().substring(0, 
3).toLowerCase() + "*";
+            assertThrows(WebloggerException.class, () -> indexManager.search(
+                FieldConstants.WEBSITE_HANDLE + ":" + handlePrefix,
+                testWeblog.getHandle(), null, testWeblog.getLocale(), 0, 
RESULTS_PER_PAGE,
+                WebloggerFactory.getWeblogger().getUrlStrategy()));
+
         } finally {
             for (WeblogEntry entry : entries) {
                 
indexManager.removeEntryIndexOperation(TestUtils.getManagedWeblogEntry(entry));
diff --git 
a/app/src/test/java/org/apache/roller/weblogger/business/search/lucene/IndexedFieldsTest.java
 
b/app/src/test/java/org/apache/roller/weblogger/business/search/lucene/IndexedFieldsTest.java
new file mode 100644
index 000000000..802977b2c
--- /dev/null
+++ 
b/app/src/test/java/org/apache/roller/weblogger/business/search/lucene/IndexedFieldsTest.java
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.roller.weblogger.business.search.lucene;
+
+import java.sql.Timestamp;
+import java.util.List;
+
+import org.apache.lucene.analysis.standard.StandardAnalyzer;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.StringField;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.store.ByteBuffersDirectory;
+import org.apache.lucene.store.Directory;
+import org.apache.roller.weblogger.pojos.User;
+import org.apache.roller.weblogger.pojos.Weblog;
+import org.apache.roller.weblogger.pojos.WeblogEntry;
+import org.apache.roller.weblogger.pojos.WeblogEntryComment;
+import org.junit.jupiter.api.Test;
+
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+/**
+ * The index holds only what search needs, and an index built by an earlier
+ * version with more fields is found so it can be rebuilt.
+ */
+class IndexedFieldsTest {
+
+    @Test
+    void entryDocumentHoldsNoAuthorOrCommenterDetails() {
+        Document doc = new IndexOperation(null) {
+            @Override
+            protected void doRun() {
+            }
+        }.getDocument(entry());
+
+        assertNotNull(doc.getField(FieldConstants.ID));
+        assertNotNull(doc.getField(FieldConstants.C_CONTENT));
+        for (String removed : FieldConstants.REMOVED_FIELDS) {
+            assertNull(doc.getField(removed), removed);
+        }
+    }
+
+    @Test
+    void indexWithARemovedFieldIsFound() throws Exception {
+        assertTrue(hasRemovedFields(FieldConstants.C_EMAIL));
+        assertFalse(hasRemovedFields(FieldConstants.TITLE));
+    }
+
+    private static boolean hasRemovedFields(String field) throws Exception {
+        try (Directory dir = new ByteBuffersDirectory()) {
+            try (IndexWriter writer = new IndexWriter(dir, new 
IndexWriterConfig(new StandardAnalyzer()))) {
+                Document doc = new Document();
+                doc.add(new StringField(field, "value", Field.Store.YES));
+                writer.addDocument(doc);
+            }
+            try (DirectoryReader reader = DirectoryReader.open(dir)) {
+                return LuceneIndexManager.hasRemovedFields(reader);
+            }
+        }
+    }
+
+    private static WeblogEntry entry() {
+        WeblogEntryComment comment = mock(WeblogEntryComment.class);
+        when(comment.getContent()).thenReturn("nice post");
+        when(comment.getName()).thenReturn("Reader");
+        when(comment.getEmail()).thenReturn("[email protected]");
+
+        Weblog weblog = mock(Weblog.class);
+        when(weblog.getHandle()).thenReturn("team");
+        User author = mock(User.class);
+        when(author.getUserName()).thenReturn("author");
+
+        WeblogEntry entry = mock(WeblogEntry.class);
+        when(entry.getId()).thenReturn("id1");
+        when(entry.getWebsite()).thenReturn(weblog);
+        when(entry.getCreator()).thenReturn(author);
+        when(entry.getTitle()).thenReturn("Title");
+        when(entry.getLocale()).thenReturn("en");
+        when(entry.getText()).thenReturn("Text");
+        Timestamp now = new Timestamp(System.currentTimeMillis());
+        when(entry.getUpdateTime()).thenReturn(now);
+        when(entry.getPubTime()).thenReturn(now);
+        when(entry.getComments()).thenReturn(List.of(comment));
+        return entry;
+    }
+}
diff --git 
a/app/src/test/java/org/apache/roller/weblogger/business/search/lucene/SearchFieldsQueryParserTest.java
 
b/app/src/test/java/org/apache/roller/weblogger/business/search/lucene/SearchFieldsQueryParserTest.java
new file mode 100644
index 000000000..047921cb5
--- /dev/null
+++ 
b/app/src/test/java/org/apache/roller/weblogger/business/search/lucene/SearchFieldsQueryParserTest.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.roller.weblogger.business.search.lucene;
+
+import org.apache.lucene.analysis.standard.StandardAnalyzer;
+import org.apache.lucene.queryparser.classic.ParseException;
+import org.apache.lucene.search.MatchAllDocsQuery;
+import org.junit.jupiter.api.Test;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+/**
+ * Searches may name only the fields they are meant to cover.
+ */
+class SearchFieldsQueryParserTest {
+
+    private static SearchOperation.SearchFieldsQueryParser parser() {
+        return new SearchOperation.SearchFieldsQueryParser(
+                new String[] {FieldConstants.CONTENT, FieldConstants.TITLE, 
FieldConstants.C_CONTENT},
+                new StandardAnalyzer());
+    }
+
+    @Test
+    void plainAndSearchFieldQueriesParse() throws Exception {
+        assertNotNull(parser().parse("roller weblog"));
+        assertNotNull(parser().parse("roll*"));
+        assertNotNull(parser().parse("title:roller"));
+        assertNotNull(parser().parse("content:roll*"));
+        assertNotNull(parser().parse("\"apache roller\"~2"));
+    }
+
+    @Test
+    void matchAllParses() throws Exception {
+        assertEquals(new MatchAllDocsQuery(), parser().parse("*:*"));
+    }
+
+    @Test
+    void otherFieldsAreRefused() {
+        for (String query : new String[] {
+                FieldConstants.C_EMAIL + ":a*",
+                FieldConstants.C_EMAIL + ":[email protected]",
+                FieldConstants.C_NAME + ":reader",
+                FieldConstants.USERNAME + ":admin",
+                FieldConstants.ANCHOR + ":[a TO z]",
+                FieldConstants.C_EMAIL + ":/re.*/",
+                FieldConstants.C_EMAIL + ":reader~",
+                "*:a*",
+                FieldConstants.C_EMAIL + ":*",
+                "roller AND " + FieldConstants.C_EMAIL + ":r?ader"}) {
+            assertThrows(ParseException.class, () -> parser().parse(query), 
query);
+        }
+    }
+}

Reply via email to