Ram has uploaded a new change for review.
https://gerrit.wikimedia.org/r/59533
Change subject: Bug: 47293 Further reduce noise in log files
......................................................................
Bug: 47293 Further reduce noise in log files
Indices mapped to a non-existent host are generating a lot of noise
in log files; bug 45795 fixes two code paths and this patch fixes the
remaining third code path.
Bug: 47293
Change-Id: Ie6be47113e57b8419d041909bff458b171b263a0
---
M src/org/wikimedia/lsearch/config/Configuration.java
M src/org/wikimedia/lsearch/config/GlobalConfiguration.java
M src/org/wikimedia/lsearch/config/IndexId.java
3 files changed, 101 insertions(+), 53 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/operations/debs/lucene-search-2
refs/changes/33/59533/1
diff --git a/src/org/wikimedia/lsearch/config/Configuration.java
b/src/org/wikimedia/lsearch/config/Configuration.java
index de23ff8..1371cfa 100644
--- a/src/org/wikimedia/lsearch/config/Configuration.java
+++ b/src/org/wikimedia/lsearch/config/Configuration.java
@@ -109,7 +109,8 @@
System.exit(1);
}
try {
- global.readFromURL(new URL(globalurl),indexpath);
+ String nullHost = getString( "Search", "nullHost", ""
); // non-existent host
+ global.readFromURL(new URL(globalurl), indexpath,
nullHost);
} catch (MalformedURLException e) {
System.out.println("Malformed URL "+globalurl+" cannot
read global configuration (check MWConfig.global in "+CONF_FILE_NAME+"),
exiting...");
System.exit(1);
diff --git a/src/org/wikimedia/lsearch/config/GlobalConfiguration.java
b/src/org/wikimedia/lsearch/config/GlobalConfiguration.java
index 5d3d419..316d341 100644
--- a/src/org/wikimedia/lsearch/config/GlobalConfiguration.java
+++ b/src/org/wikimedia/lsearch/config/GlobalConfiguration.java
@@ -123,6 +123,9 @@
/** Whether to report warnings and info */
protected static boolean verbose = true;
+ /** name of non-existent host used to disable indices */
+ protected String nullHost;
+
/** Sections in lsearch-config.conf */
protected static enum Section { DATABASE, INDEX, SEARCH, INDEXPATH,
NAMESPACE_PREFIX, OAI, DATABASE_GROUP, NAMESPACE_BOOST };
@@ -333,13 +336,15 @@
* @param url
* @throws IOException
*/
- public void readFromURL(URL url, String indexpath) throws IOException{
+ public void readFromURL(URL url, String indexpath, String nullH )
throws IOException{
+ this.nullHost = nullH;
BufferedReader in;
try {
in = new BufferedReader(
new InputStreamReader(
url.openStream()));
read(in,indexpath);
+
} catch (IOException e) {
System.out.println("I/O Error in opening or reading
global config at url "+url);
throw e;
@@ -741,6 +746,15 @@
// HashSet<String> mySearchHosts =
getMySearchHosts(dbname,dbrole);
HashSet<String> mySearchHosts = searchHosts;
boolean mySearch =
searchHosts.contains(hostAddr) || searchHosts.contains(hostName);
+
+ // this index is disabled if the only host that
searches it is the null host
+ boolean disabled =
+ "" != nullHost
+ && searchHosts.contains( nullHost )
+ && mySearchHosts.contains( nullHost )
+ && 1 == searchHosts.size()
+ && 1 == mySearchHosts.size();
+
String indexHost = getIndexHost(dbrole,dbname);
boolean myIndex = isMyHost(indexHost);
Hashtable<String,String> typeidParams =
database.get(dbname).get(typeid);
@@ -766,41 +780,43 @@
String oairepo = getOAIRepo(dbname);
IndexId iid = new IndexId(dbrole,
- type,
- indexHost,
-
rsyncIndexPath,
-
database.get(dbname).get(type),
-
typeidParams,
- searchHosts,
-
mySearchHosts,
- indexPath,
- myIndex,
- mySearch,
- oairepo,
-
isSubdivided,
-
dbnameTitlesPart.get(dbname),
-
dbnameSuffix.get(dbname),
-
getSuffixToDBMap(dbrole,dbnameTitlesPart,dbnameSuffix));
+ type,
+ indexHost,
+ rsyncIndexPath,
+
database.get(dbname).get(type),
+ typeidParams,
+ searchHosts,
+ mySearchHosts,
+ indexPath,
+ myIndex,
+ mySearch,
+ oairepo,
+ isSubdivided,
+
dbnameTitlesPart.get(dbname),
+
dbnameSuffix.get(dbname),
+
getSuffixToDBMap(dbrole,dbnameTitlesPart,dbnameSuffix),
+ disabled);
indexIdPool.put(dbrole,iid);
// add precursor indexes
if(type.equals("spell") ||
type.equals("prefix")){
iid = new IndexId(dbrole+".pre",
- "precursor",
- indexHost,
- rsyncIndexPath,
- database.get(dbname).get(type),
- typeidParams,
- new HashSet<String>(), // precursors cannot be
searched
- new HashSet<String>(),
- indexPath,
- myIndex,
- false,
- oairepo,
- isSubdivided,
- dbnameTitlesPart.get(dbname),
- dbnameSuffix.get(dbname),
-
getSuffixToDBMap(dbrole,dbnameTitlesPart,dbnameSuffix));
+ "precursor",
+ indexHost,
+ rsyncIndexPath,
+
database.get(dbname).get(type),
+ typeidParams,
+ new
HashSet<String>(), // precursors cannot be searched
+ new HashSet<String>(),
+ indexPath,
+ myIndex,
+ false,
+ oairepo,
+ isSubdivided,
+
dbnameTitlesPart.get(dbname),
+
dbnameSuffix.get(dbname),
+
getSuffixToDBMap(dbrole,dbnameTitlesPart,dbnameSuffix),
+ false);
indexIdPool.put(dbrole+".pre",iid);
}
// add highlight indexes
@@ -811,23 +827,32 @@
mySearchHosts = searchHosts;
// mySearchHosts =
getMySearchHosts(dbname,dbrole);
mySearch =
searchHosts.contains(hostAddr) || searchHosts.contains(hostName);
+ // this index is disabled if the only
host that searches it is the null host
+ disabled =
+ "" != nullHost
+ && searchHosts.contains(
nullHost )
+ && mySearchHosts.contains(
nullHost )
+ && 1 == searchHosts.size()
+ && 1 == mySearchHosts.size();
+
// other options are identical!
iid = new IndexId(dbrole,
- type,
- indexHost,
-
rsyncIndexPath,
-
database.get(dbname).get(type),
-
typeidParams,
- searchHosts,
-
mySearchHosts,
- indexPath,
- myIndex,
- mySearch,
- oairepo,
-
isSubdivided,
-
dbnameTitlesPart.get(dbname),
-
dbnameSuffix.get(dbname),
-
getSuffixToDBMap(dbrole,dbnameTitlesPart,dbnameSuffix));
+ type,
+ indexHost,
+ rsyncIndexPath,
+
database.get(dbname).get(type),
+ typeidParams,
+ searchHosts,
+ mySearchHosts,
+ indexPath,
+ myIndex,
+ mySearch,
+ oairepo,
+ isSubdivided,
+
dbnameTitlesPart.get(dbname),
+
dbnameSuffix.get(dbname),
+
getSuffixToDBMap(dbrole,dbnameTitlesPart,dbnameSuffix),
+ disabled);
indexIdPool.put(dbrole,iid);
}
diff --git a/src/org/wikimedia/lsearch/config/IndexId.java
b/src/org/wikimedia/lsearch/config/IndexId.java
index 5bdfb28..ed315bb 100644
--- a/src/org/wikimedia/lsearch/config/IndexId.java
+++ b/src/org/wikimedia/lsearch/config/IndexId.java
@@ -48,6 +48,8 @@
protected HashSet<String> mySearchHosts;
/** If this hosts searches the index */
protected boolean mySearch;
+ /** If this index is disabled by mapping to a non-existent host */
+ protected boolean disabled;
/** Database name, e.g. for entest.mainpart, it's entest */
protected String dbname;
@@ -193,11 +195,25 @@
* @param myIndex if this machines is an indexer for this index
* @param mySearch if this machine is a searcher for this index
*/
- public IndexId(String dbrole, String type, String indexHost, String
indexRsyncPath,
- Hashtable<String, String> typeParams, Hashtable<String,
String> params,
- HashSet<String> searchHosts, HashSet<String>
mySearchHosts, String localIndexPath,
- boolean myIndex, boolean mySearch, String
OAIRepository, boolean isSubdivided,
- String titlesIndex, String titlesSuffix,
Hashtable<String,String> suffixToDbname) {
+ public IndexId(String dbrole,
+ String type,
+ String indexHost,
+ String indexRsyncPath,
+ Hashtable<String, String> typeParams,
+ Hashtable<String, String> params,
+ HashSet<String> searchHosts,
+ HashSet<String> mySearchHosts,
+ String localIndexPath,
+ boolean myIndex,
+ boolean mySearch,
+ String OAIRepository,
+ boolean isSubdivided,
+ String titlesIndex,
+ String titlesSuffix,
+ Hashtable<String,String> suffixToDbname,
+ boolean dis)
+ {
+ disabled = dis;
final String sep = Configuration.PATH_SEP;
this.indexHost = indexHost;
if(!indexRsyncPath.endsWith("/"))
@@ -859,9 +875,15 @@
}
throw new RuntimeException("Called getInterwikiBySuffix() on
non-titles_by_suffix index");
}
+
+ /** If index is disabled */
+ public boolean isDisabled() {
+ return disabled;
+ }
+
/** If index containing only titles is defined */
public boolean hasTitlesIndex(){
- return titlesIndex != null;
+ return titlesIndex != null && ! disabled;
}
/** Get iid where the titles for this index are */
public IndexId getTitlesIndex() {
--
To view, visit https://gerrit.wikimedia.org/r/59533
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: Ie6be47113e57b8419d041909bff458b171b263a0
Gerrit-PatchSet: 1
Gerrit-Project: operations/debs/lucene-search-2
Gerrit-Branch: master
Gerrit-Owner: Ram <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
