This is an automated email from the ASF dual-hosted git repository. janhoy pushed a commit to tag history/branches/lucene-solr/jira/solr-12730 in repository https://gitbox.apache.org/repos/asf/solr.git
commit 2bff40112b91eeec3d3a7b2abfe7d92aa84f5586 Author: Shalin Shekhar Mangar <[email protected]> AuthorDate: Thu Nov 1 11:09:16 2018 +0530 Adding more logging around when files are reference counted Added custom CMS impl which sets thread names with MDC context information --- .../org/apache/lucene/index/IndexFileDeleter.java | 23 +++++++++++++++-- .../org/apache/solr/update/SolrIndexConfig.java | 29 +++++++++++++++++++++- .../processor/DistributedUpdateProcessor.java | 1 - 3 files changed, 49 insertions(+), 4 deletions(-) diff --git a/lucene/core/src/java/org/apache/lucene/index/IndexFileDeleter.java b/lucene/core/src/java/org/apache/lucene/index/IndexFileDeleter.java index f76cb98..cd5cf7f 100644 --- a/lucene/core/src/java/org/apache/lucene/index/IndexFileDeleter.java +++ b/lucene/core/src/java/org/apache/lucene/index/IndexFileDeleter.java @@ -20,6 +20,8 @@ package org.apache.lucene.index; import java.io.Closeable; import java.io.FileNotFoundException; import java.io.IOException; +import java.io.PrintWriter; +import java.io.StringWriter; import java.nio.file.NoSuchFileException; import java.util.ArrayList; import java.util.Collection; @@ -564,7 +566,16 @@ final class IndexFileDeleter implements Closeable { RefCount rc = getRefCount(fileName); if (infoStream.isEnabled("IFD")) { if (VERBOSE_REF_COUNTS) { - infoStream.message("IFD", " IncRef \"" + fileName + "\": pre-incr count is " + rc.count); + // todo nocommit remove the extra logging + String extra = null; + if ("_0.fdt".equals(fileName)) { + StringWriter stringWriter = new StringWriter(); + PrintWriter writer = new PrintWriter(stringWriter); + new Exception().printStackTrace(writer); + extra = stringWriter.toString(); + } + + infoStream.message("IFD", " IncRef \"" + fileName + "\": pre-incr count is " + rc.count + (extra != null ? " and stack is " + extra : "")); } } rc.IncRef(); @@ -602,7 +613,15 @@ final class IndexFileDeleter implements Closeable { RefCount rc = getRefCount(fileName); if (infoStream.isEnabled("IFD")) { if (VERBOSE_REF_COUNTS) { - infoStream.message("IFD", " DecRef \"" + fileName + "\": pre-decr count is " + rc.count); + String extra = null; + // todo nocommit remove the extra logging + if ("_0.fdt".equals(fileName)) { + StringWriter stringWriter = new StringWriter(); + PrintWriter writer = new PrintWriter(stringWriter); + new Exception().printStackTrace(writer); + extra = stringWriter.toString(); + } + infoStream.message("IFD", " DecRef \"" + fileName + "\": pre-decr count is " + rc.count + (extra != null ? " and stack is " + extra : "")); } } if (rc.DecRef() == 0) { diff --git a/solr/core/src/java/org/apache/solr/update/SolrIndexConfig.java b/solr/core/src/java/org/apache/solr/update/SolrIndexConfig.java index 48b2417..38f16c2 100644 --- a/solr/core/src/java/org/apache/solr/update/SolrIndexConfig.java +++ b/solr/core/src/java/org/apache/solr/update/SolrIndexConfig.java @@ -18,6 +18,7 @@ package org.apache.solr.update; import java.io.IOException; import java.lang.invoke.MethodHandles; +import java.util.Collection; import java.util.Collections; import java.util.List; import java.util.Map; @@ -25,6 +26,7 @@ import java.util.Map; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.DelegatingAnalyzerWrapper; import org.apache.lucene.index.ConcurrentMergeScheduler; +import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriter.IndexReaderWarmer; import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.index.MergePolicy; @@ -48,6 +50,7 @@ import org.apache.solr.schema.IndexSchema; import org.apache.solr.util.SolrPluginUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.slf4j.MDC; import static org.apache.solr.core.Config.assertWarnOrFail; @@ -283,7 +286,7 @@ public class SolrIndexConfig implements MapSerializable { private MergeScheduler buildMergeScheduler(IndexSchema schema) { String msClassName = mergeSchedulerInfo == null ? SolrIndexConfig.DEFAULT_MERGE_SCHEDULER_CLASSNAME : mergeSchedulerInfo.className; - MergeScheduler scheduler = schema.getResourceLoader().newInstance(msClassName, MergeScheduler.class); + MergeScheduler scheduler = new TestCMS(); // todo nocommit if (mergeSchedulerInfo != null) { // LUCENE-5080: these two setters are removed, so we have to invoke setMaxMergesAndThreads @@ -312,4 +315,28 @@ public class SolrIndexConfig implements MapSerializable { return scheduler; } + static class TestCMS extends ConcurrentMergeScheduler { + @Override + protected synchronized MergeThread getMergeThread(IndexWriter writer, MergePolicy.OneMerge merge) throws IOException { + MergeThread mergeThread = super.getMergeThread(writer, merge); + final Map<String, String> submitterContext = MDC.getCopyOfContextMap(); + StringBuilder contextString = new StringBuilder(); + if (submitterContext != null) { + Collection<String> values = submitterContext.values(); + + for (String value : values) { + contextString.append(value + " "); + } + if (contextString.length() > 1) { + contextString.setLength(contextString.length() - 1); + } + } + + String ctxStr = contextString.toString().replace("/", "//"); + final String submitterContextStr = ctxStr.length() <= 512 ? ctxStr : ctxStr.substring(0, 512); + mergeThread.setName(mergeThread.getName() + "-processing-" + submitterContextStr); + return mergeThread; + } + } + } diff --git a/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java b/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java index cc0f8fb..8bdc8bc 100644 --- a/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java +++ b/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java @@ -1388,7 +1388,6 @@ public class DistributedUpdateProcessor extends UpdateRequestProcessor { boolean dropCmd = false; if (!forwardToLeader) { dropCmd = versionDelete(cmd); - System.out.println("Doing versionDelete on " + req.getCore().getName() + " dropCmd=" + dropCmd); } if (dropCmd) {
