This is an automated email from the ASF dual-hosted git repository.
gerlowskija pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/solr-sandbox.git
The following commit(s) were added to refs/heads/main by this push:
new e978a7a Ignore producer URP (if disabled) in standalone mode (#79)
e978a7a is described below
commit e978a7a4c1349cfcd6511663a2b895b0e972c921
Author: Jason Gerlowski <[email protected]>
AuthorDate: Thu Oct 19 14:52:59 2023 -0400
Ignore producer URP (if disabled) in standalone mode (#79)
XDC really only makes sense in a SolrCloud context. But all the same,
its nice to handle MirroringUpdateRequestProcessorFactory more
gracefully if someone does accidentally configure their standalone core
to use the XDC producer URP.
This commit tweaks MirroringUpdateRequestProcessorFactory's
initialization so that in standalone mode it emits a warning about the
misconfiguration if disabled, and fails core init/load entirely if the
Mirroring URP is 'enabled=true'.
---
.../MirroringUpdateRequestProcessorFactory.java | 25 +++--
.../crossdc/CrossDCProducerSolrStandaloneTest.java | 101 +++++++++++++++++
.../conf/solrconfig-producerdisabled.xml | 119 +++++++++++++++++++++
3 files changed, 239 insertions(+), 6 deletions(-)
diff --git
a/crossdc-producer/src/main/java/org/apache/solr/update/processor/MirroringUpdateRequestProcessorFactory.java
b/crossdc-producer/src/main/java/org/apache/solr/update/processor/MirroringUpdateRequestProcessorFactory.java
index 0077339..80ce2a1 100644
---
a/crossdc-producer/src/main/java/org/apache/solr/update/processor/MirroringUpdateRequestProcessorFactory.java
+++
b/crossdc-producer/src/main/java/org/apache/solr/update/processor/MirroringUpdateRequestProcessorFactory.java
@@ -96,6 +96,7 @@ public class MirroringUpdateRequestProcessorFactory extends
UpdateRequestProcess
}
}
+
private static class MyCloseHook extends CloseHook {
private final Closer closer;
@@ -129,10 +130,7 @@ public class MirroringUpdateRequestProcessorFactory
extends UpdateRequestProcess
}
- @Override
- public void inform(SolrCore core) {
- log.info("KafkaRequestMirroringHandler inform enabled={}",
this.enabled);
-
+ private void lookupPropertyOverridesInZk(SolrCore core) {
log.info("Producer startup config properties before adding additional
properties from Zookeeper={}", properties);
try {
@@ -140,7 +138,7 @@ public class MirroringUpdateRequestProcessorFactory extends
UpdateRequestProcess
ConfUtil.fillProperties(solrZkClient, properties);
applyArgsOverrides();
CollectionProperties cp = new CollectionProperties(solrZkClient);
- Map<String,String> collectionProperties =
cp.getCollectionProperties(core.getCoreDescriptor().getCollectionName());
+ Map<String,String> collectionProperties =
cp.getCollectionProperties(core.getCoreDescriptor().getCollectionName());
for (ConfigProperty configKey :
KafkaCrossDcConf.CONFIG_PROPERTIES) {
String val = collectionProperties.get("crossdc." +
configKey.getKey());
if (val != null && !val.isBlank()) {
@@ -159,6 +157,21 @@ public class MirroringUpdateRequestProcessorFactory
extends UpdateRequestProcess
log.error("Exception looking for CrossDC configuration in
Zookeeper", e);
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
"Exception looking for CrossDC configuration in Zookeeper", e);
}
+ }
+
+ @Override
+ public void inform(SolrCore core) {
+ log.info("KafkaRequestMirroringHandler inform enabled={}",
this.enabled);
+
+ if (core.getCoreContainer().isZooKeeperAware()) {
+ lookupPropertyOverridesInZk(core);
+ } else {
+ applyArgsOverrides();
+ if (enabled) {
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
getClass().getSimpleName() + " only supported in SolrCloud mode; please disable
or remove from solrconfig.xml");
+ }
+ log.warn("Core '{}' was configured to use a disabled {}, but {} is
only supported in SolrCloud deployments. A NoOp processor will be used
instead", core.getName(), this.getClass().getSimpleName(),
this.getClass().getSimpleName());
+ }
if (!enabled) {
return;
@@ -240,7 +253,7 @@ public class MirroringUpdateRequestProcessorFactory extends
UpdateRequestProcess
DistribPhase.parseParam(req.getParams().get(DISTRIB_UPDATE_PARAM)), doMirroring
? mirroringHandler : null);
}
- private static class NoOpUpdateRequestProcessor extends
UpdateRequestProcessor {
+ public static class NoOpUpdateRequestProcessor extends
UpdateRequestProcessor {
NoOpUpdateRequestProcessor(UpdateRequestProcessor next) {
super(next);
}
diff --git
a/crossdc-producer/src/test/java/org/apache/solr/crossdc/CrossDCProducerSolrStandaloneTest.java
b/crossdc-producer/src/test/java/org/apache/solr/crossdc/CrossDCProducerSolrStandaloneTest.java
new file mode 100644
index 0000000..8a0c3fd
--- /dev/null
+++
b/crossdc-producer/src/test/java/org/apache/solr/crossdc/CrossDCProducerSolrStandaloneTest.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.crossdc;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.solr.SolrTestCaseJ4;
+import org.apache.solr.client.solrj.embedded.EmbeddedSolrServer;
+import org.apache.solr.core.SolrCoreInitializationException;
+import org.apache.solr.update.processor.MirroringUpdateRequestProcessorFactory;
+import org.junit.After;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+
+/**
+ * Unit test validating that {@link MirroringUpdateRequestProcessorFactory}
responds appropriately in Solr is running in standalone mode.
+ */
+public class CrossDCProducerSolrStandaloneTest extends SolrTestCaseJ4 {
+
+ @After
+ public void tearDown() throws Exception {
+ super.tearDown();
+ h.close();
+ }
+
+ @Test
+ public void testSolrStandaloneQuietlyNoopsDisabledProducer() throws
Exception {
+ try (final EmbeddedSolrServer server =
createProducerCoreWithProperties("solrconfig-producerdisabled.xml",
"producerDisabledCore")) {
+ try (final var producerDisabledCore =
server.getCoreContainer().getCore("producerDisabledCore")) {
+ assertNotNull(producerDisabledCore);
+ final var mirrorChain =
producerDisabledCore.getUpdateProcessingChain("mirrorUpdateChain");
+ assertNotNull(mirrorChain);
+ final var updateProcessorList = mirrorChain.getProcessors();
+ final var mirroringFactoryOption = updateProcessorList.stream()
+ .filter(pf -> pf instanceof
MirroringUpdateRequestProcessorFactory)
+ .findFirst();
+ assertTrue("No mirroring factory found in " +
updateProcessorList, mirroringFactoryOption.isPresent());
+ final var mirroringFactory = mirroringFactoryOption.get();
+
+ final var mirroringProcessorInstance =
mirroringFactory.getInstance(null, null, null);
+
assertEquals(MirroringUpdateRequestProcessorFactory.NoOpUpdateRequestProcessor.class,
mirroringProcessorInstance.getClass());
+ }
+ }
+ }
+
+ @Test
+ public void testEnabledProcessorFailsCoreInitInSolrStandalone() throws
Exception {
+ try (final EmbeddedSolrServer server =
createProducerCoreWithProperties("solrconfig.xml", "producerEnabledCore")) {
+ expectThrows(SolrCoreInitializationException.class, () -> {
+ final var core =
server.getCoreContainer().getCore("producerEnabledCore");
+ // Should be preempted by exception in line above, but ensures
the core is closed in case the test is about to fail
+ core.close();
+ });
+ }
+ }
+
+ private static EmbeddedSolrServer createProducerCoreWithProperties(String
solrConfigName, String coreName) throws Exception {
+ Path tmpHome = createTempDir("tmp-home");
+ Path coreDir = tmpHome.resolve(coreName);
+ populateCoreDirectory("src/test/resources/configs/cloud-minimal/conf",
solrConfigName, coreDir.toFile());
+ initCore(
+ "solrconfig.xml", "schema.xml",
tmpHome.toAbsolutePath().toString(), coreName);
+
+ return new EmbeddedSolrServer(h.getCoreContainer(), coreName);
+ }
+
+ /**
+ * Copy configset files to a specified location
+ *
+ * @param sourceLocation the location of schema and solrconfig files to
copy
+ * @param solrConfigName the name of the solrconfig file to use for this
core
+ * @param coreDirectory an empty preexisting location use as a core
directory.
+ * @throws IOException
+ */
+ private static void populateCoreDirectory(String sourceLocation, String
solrConfigName, File coreDirectory) throws IOException {
+ File subHome = new File(coreDirectory, "conf");
+ if (! coreDirectory.exists()) {
+ assertTrue("Failed to make subdirectory ", coreDirectory.mkdirs());
+ }
+ Files.createFile(coreDirectory.toPath().resolve("core.properties"));
+ FileUtils.copyFile(new File(sourceLocation, "schema.xml"), new
File(subHome, "schema.xml"));
+ FileUtils.copyFile(new File(sourceLocation, solrConfigName), new
File(subHome, "solrconfig.xml"));
+ }
+}
diff --git
a/crossdc-producer/src/test/resources/configs/cloud-minimal/conf/solrconfig-producerdisabled.xml
b/crossdc-producer/src/test/resources/configs/cloud-minimal/conf/solrconfig-producerdisabled.xml
new file mode 100644
index 0000000..8de9263
--- /dev/null
+++
b/crossdc-producer/src/test/resources/configs/cloud-minimal/conf/solrconfig-producerdisabled.xml
@@ -0,0 +1,119 @@
+<?xml version="1.0" ?>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<!-- Minimal solrconfig.xml with /select, /admin and /update only -->
+
+<config>
+
+ <dataDir>${solr.data.dir:}</dataDir>
+
+ <directoryFactory name="DirectoryFactory"
+
class="${directoryFactory:solr.NRTCachingDirectoryFactory}"/>
+ <schemaFactory class="ClassicIndexSchemaFactory"/>
+
+ <luceneMatchVersion>${tests.luceneMatchVersion:LATEST}</luceneMatchVersion>
+
+ <indexConfig>
+ <mergePolicyFactory
class="${mergePolicyFactory:org.apache.solr.index.TieredMergePolicyFactory}">
+ <int name="maxMergeAtOnce">${maxMergeAtOnce:10}</int>
+ <int name="segmentsPerTier">${segmentsPerTier:10}</int>
+ <double name="noCFSRatio">${noCFSRatio:.1}</double>
+ </mergePolicyFactory>
+
+ <useCompoundFile>${useCompoundFile:true}</useCompoundFile>
+
+ <ramBufferSizeMB>${ramBufferSizeMB:160}</ramBufferSizeMB>
+ <maxBufferedDocs>${maxBufferedDocs:250000}</maxBufferedDocs> <!--
Force the common case to flush by doc count -->
+ <!-- <ramPerThreadHardLimitMB>60</ramPerThreadHardLimitMB> -->
+
+ <!-- <mergeScheduler
class="org.apache.lucene.index.ConcurrentMergeScheduler">
+ <int name="maxThreadCount">6</int>
+ <int name="maxMergeCount">8</int>
+ <bool name="ioThrottle">false</bool>
+ </mergeScheduler> -->
+
+ <writeLockTimeout>1000</writeLockTimeout>
+ <commitLockTimeout>10000</commitLockTimeout>
+
+ <!-- this sys property is not set by SolrTestCaseJ4 because almost all
tests should
+ use the single process lockType for speed - but tests that explicitly
need
+ to vary the lockType canset it as needed.
+ -->
+ <lockType>${lockType:single}</lockType>
+
+ <infoStream>${infostream:false}</infoStream>
+
+ </indexConfig>
+
+ <updateHandler class="solr.DirectUpdateHandler2">
+ <commitWithin>
+ <softCommit>${commitwithin.softcommit:true}</softCommit>
+ </commitWithin>
+ <autoCommit>
+ <maxTime>${autoCommit.maxTime:60000}</maxTime>
+ </autoCommit>
+ <updateLog class="${ulog:solr.UpdateLog}"
enable="${enable.update.log:true}"/>
+ </updateHandler>
+
+ <requestHandler name="/select" class="solr.SearchHandler">
+ <lst name="defaults">
+ <str name="echoParams">explicit</str>
+ <str name="indent">true</str>
+ <str name="df">text</str>
+ </lst>
+
+ </requestHandler>
+
+ <query>
+ <queryResultCache
+ enabled="${queryResultCache.enabled:false}"
+ class="${queryResultCache.class:solr.CaffeineCache}"
+ size="${queryResultCache.size:0}"
+ initialSize="${queryResultCache.initialSize:0}"
+ autowarmCount="${queryResultCache.autowarmCount:0}"/>
+ <documentCache
+ enabled="${documentCache.enabled:false}"
+ class="${documentCache.class:solr.CaffeineCache}"
+ size="${documentCache.size:0}"
+ initialSize="${documentCache.initialSize:0}"
+ autowarmCount="${documentCache.autowarmCount:0}"/>
+ <filterCache
+ enabled ="${filterCache.enabled:false}"
+ class="${filterCache.class:solr.CaffeineCache}"
+ size="${filterCache.size:1}"
+ initialSize="${filterCache.initialSize:1}"
+ autowarmCount="${filterCache.autowarmCount:0}"
+ async="${filterCache.async:false}"/>
+ <cache name="myPerSegmentCache"
+ enabled="${myPerSegmentCache.enabled:false}"
+ class="${myPerSegmentCache.class:solr.CaffeineCache}"
+ size="${myPerSegmentCache.size:0}"
+ initialSize="${myPerSegmentCache.initialSize:0}"
+ autowarmCount="${myPerSegmentCache.autowarmCount:0}"/>
+ </query>
+
+ <updateRequestProcessorChain name="mirrorUpdateChain" default="true">
+ <processor
class="org.apache.solr.update.processor.MirroringUpdateRequestProcessorFactory">
+ <bool name="enabled">false</bool>
+ </processor>
+ <processor class="solr.LogUpdateProcessorFactory" />
+ <processor class="solr.RunUpdateProcessorFactory" />
+ </updateRequestProcessorChain>
+
+</config>
