This is an automated email from the ASF dual-hosted git repository.
dsmiley pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/solr.git
The following commit(s) were added to refs/heads/main by this push:
new edc3282 SOLR-15258: fix ConfigSetService to support bootstrap_conf
(#322)
edc3282 is described below
commit edc3282e333f7bc47471b69eeb73184da14ffcae
Author: Nazerke Seidan <[email protected]>
AuthorDate: Thu Oct 7 17:39:42 2021 +0200
SOLR-15258: fix ConfigSetService to support bootstrap_conf (#322)
(unreleased regression)
Wasn't previously tested; now we have a test.
If a ZK chroot is used as well, user needs to create it; won't be
auto-created.
The Thread.sleep wasn't needed.
Moved more logic out of ZkController.
Co-authored-by: Nazerke Seidan <[email protected]>
---
.../org/apache/solr/core/ConfigSetService.java | 57 ++++++++++++++++------
.../src/java/org/apache/solr/core/ZkContainer.java | 36 +++-----------
.../solr/common/cloud/TestZkConfigSetService.java | 24 +++++++++
3 files changed, 73 insertions(+), 44 deletions(-)
diff --git a/solr/core/src/java/org/apache/solr/core/ConfigSetService.java
b/solr/core/src/java/org/apache/solr/core/ConfigSetService.java
index de80fba..5452648 100644
--- a/solr/core/src/java/org/apache/solr/core/ConfigSetService.java
+++ b/solr/core/src/java/org/apache/solr/core/ConfigSetService.java
@@ -53,14 +53,10 @@ public abstract class ConfigSetService {
private static final Logger log =
LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
public static ConfigSetService createConfigSetService(CoreContainer
coreContainer) {
- final ConfigSetService configSetService = instantiate(coreContainer);
- try {
- bootstrapDefaultConfigSet(configSetService);
- } catch (UnsupportedOperationException e) {
- log.info("_default config couldn't be uploaded");
- } catch (IOException e) {
- throw new SolrException(
- SolrException.ErrorCode.SERVER_ERROR, "_default config couldn't
be uploaded ", e);
+ ConfigSetService configSetService = instantiate(coreContainer);
+ // bootstrap conf in SolrCloud mode
+ if (coreContainer.getZkController() != null) {
+ configSetService.bootstrapConfigSet(coreContainer);
}
return configSetService;
}
@@ -87,20 +83,53 @@ public abstract class ConfigSetService {
}
}
- private static void bootstrapDefaultConfigSet(ConfigSetService
configSetService) throws IOException {
- if (configSetService.checkConfigExists("_default") == false) {
+ private void bootstrapConfigSet(CoreContainer coreContainer) {
+ // bootstrap _default conf, bootstrap_confdir and bootstrap_conf if
provided via system property
+ try {
+ // _default conf
+ bootstrapDefaultConf();
+
+ // bootstrap_confdir
+ String confDir = System.getProperty("bootstrap_confdir");
+ if (confDir != null) {
+ bootstrapConfDir(confDir);
+ }
+
+ // bootstrap_conf
+ boolean boostrapConf = Boolean.getBoolean("bootstrap_conf");
+ if (boostrapConf == true) {
+ bootstrapConf(coreContainer);
+ }
+ } catch (IOException e) {
+ throw new SolrException(
+ SolrException.ErrorCode.SERVER_ERROR, "Config couldn't be uploaded
", e);
+ }
+ }
+
+ private void bootstrapDefaultConf() throws IOException {
+ if (this.checkConfigExists("_default") == false) {
String configDirPath = getDefaultConfigDirPath();
if (configDirPath == null) {
log.warn(
- "The _default configset could not be uploaded. Please provide
'solr.default.confdir' parameter that points to a configset {} {}",
- "intended to be the default. Current 'solr.default.confdir'
value:",
-
System.getProperty(SolrDispatchFilter.SOLR_DEFAULT_CONFDIR_ATTRIBUTE));
+ "The _default configset could not be uploaded. Please provide
'solr.default.confdir' parameter that points to a configset {} {}",
+ "intended to be the default. Current 'solr.default.confdir'
value:",
+
System.getProperty(SolrDispatchFilter.SOLR_DEFAULT_CONFDIR_ATTRIBUTE));
} else {
-
configSetService.uploadConfig(ConfigSetsHandler.DEFAULT_CONFIGSET_NAME,
Paths.get(configDirPath));
+ this.uploadConfig(ConfigSetsHandler.DEFAULT_CONFIGSET_NAME,
Paths.get(configDirPath));
}
}
}
+ private void bootstrapConfDir(String confDir) throws IOException {
+ Path configPath = Paths.get(confDir);
+ if (!Files.isDirectory(configPath)) {
+ throw new IllegalArgumentException ("bootstrap_confdir must be a
directory of configuration files, configPath: " + configPath);
+ }
+ String confName = System.getProperty(
+ ZkController.COLLECTION_PARAM_PREFIX +
ZkController.CONFIGNAME_PROP, "configuration1");
+ this.uploadConfig(confName, configPath);
+ }
+
/**
* Gets the absolute filesystem path of the _default configset to bootstrap
from. First tries the
* sysprop "solr.default.confdir". If not found, tries to find the _default
dir relative to the
diff --git a/solr/core/src/java/org/apache/solr/core/ZkContainer.java
b/solr/core/src/java/org/apache/solr/core/ZkContainer.java
index cb268c3..0fb5009 100644
--- a/solr/core/src/java/org/apache/solr/core/ZkContainer.java
+++ b/solr/core/src/java/org/apache/solr/core/ZkContainer.java
@@ -16,11 +16,14 @@
*/
package org.apache.solr.core;
+import static org.apache.solr.common.cloud.UrlScheme.HTTP;
+import static org.apache.solr.common.cloud.UrlScheme.HTTPS;
+import static org.apache.solr.common.cloud.UrlScheme.HTTPS_PORT_PROP;
+import static org.apache.solr.common.cloud.ZkStateReader.URL_SCHEME;
+
import java.io.File;
import java.io.IOException;
import java.lang.invoke.MethodHandles;
-import java.nio.file.Files;
-import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.List;
import java.util.concurrent.ExecutorService;
@@ -37,7 +40,6 @@ import org.apache.solr.common.SolrException;
import org.apache.solr.common.cloud.ClusterProperties;
import org.apache.solr.common.cloud.Replica;
import org.apache.solr.common.cloud.UrlScheme;
-import org.apache.solr.common.cloud.ZkMaintenanceUtils;
import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.cloud.ZooKeeperException;
import org.apache.solr.common.util.ExecutorUtil;
@@ -47,11 +49,6 @@ import org.apache.zookeeper.KeeperException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import static org.apache.solr.common.cloud.UrlScheme.HTTP;
-import static org.apache.solr.common.cloud.UrlScheme.HTTPS;
-import static org.apache.solr.common.cloud.UrlScheme.HTTPS_PORT_PROP;
-import static org.apache.solr.common.cloud.ZkStateReader.URL_SCHEME;
-
/**
* Used by {@link CoreContainer} to hold ZooKeeper / SolrCloud info,
especially {@link ZkController}.
* Mainly it does some ZK initialization, and ensures a loading core registers
in ZK.
@@ -120,14 +117,12 @@ public class ZkContainer {
} else {
log.info("Zookeeper client={}", zookeeperHost);
}
- String confDir = System.getProperty("bootstrap_confdir");
- boolean boostrapConf = Boolean.getBoolean("bootstrap_conf");
boolean createRoot = Boolean.getBoolean("createZkChroot");
// We may have already loaded NodeConfig from zookeeper with same
connect string, so no need to recheck chroot
boolean alreadyUsedChroot = (cc.getConfig().isFromZookeeper()
&&
zookeeperHost.equals(cc.getConfig().getDefaultZkHost()));
- if(!alreadyUsedChroot && !ZkController.checkChrootPath(zookeeperHost,
(confDir!=null) || boostrapConf || zkRunOnly || createRoot)) {
+ if(!alreadyUsedChroot && !ZkController.checkChrootPath(zookeeperHost,
zkRunOnly || createRoot)) {
throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR,
"A chroot was specified in ZkHost but the znode doesn't exist. "
+ zookeeperHost);
}
@@ -145,25 +140,6 @@ public class ZkContainer {
} else {
UrlScheme.INSTANCE.setUrlScheme(System.getProperty(URL_SCHEME,
HTTP));
}
-
- if (zkServer.getServers().size() > 1 && confDir == null &&
boostrapConf == false) {
- // we are part of an ensemble and we are not uploading the config
- pause to give the config time
- // to get up
- Thread.sleep(10000);
- }
- }
-
- if(confDir != null) {
- Path configPath = Paths.get(confDir);
- if (!Files.isDirectory(configPath))
- throw new IllegalArgumentException("bootstrap_confdir must be a
directory of configuration files");
-
- String confName =
System.getProperty(ZkController.COLLECTION_PARAM_PREFIX+ZkController.CONFIGNAME_PROP,
"configuration1");
- ZkMaintenanceUtils.uploadToZK(zkController.getZkClient(),
configPath, ZkMaintenanceUtils.CONFIGS_ZKNODE + "/" + confName,
ZkMaintenanceUtils.UPLOAD_FILENAME_EXCLUDE_PATTERN);
- }
-
- if(boostrapConf) {
- ConfigSetService.bootstrapConf(cc);
}
this.zkController = zkController;
diff --git
a/solr/solrj/src/test/org/apache/solr/common/cloud/TestZkConfigSetService.java
b/solr/solrj/src/test/org/apache/solr/common/cloud/TestZkConfigSetService.java
index 58facb5..c2c7975 100644
---
a/solr/solrj/src/test/org/apache/solr/common/cloud/TestZkConfigSetService.java
+++
b/solr/solrj/src/test/org/apache/solr/common/cloud/TestZkConfigSetService.java
@@ -17,10 +17,13 @@
package org.apache.solr.common.cloud;
import com.google.common.base.Throwables;
+import org.apache.solr.SolrJettyTestBase;
import org.apache.solr.SolrTestCaseJ4;
+import org.apache.solr.cloud.AbstractZkTestCase;
import org.apache.solr.cloud.ZkConfigSetService;
import org.apache.solr.cloud.ZkTestServer;
import org.apache.solr.core.ConfigSetService;
+import org.apache.solr.core.CoreContainer;
import org.apache.zookeeper.KeeperException;
import org.apache.zookeeper.ZooDefs;
import org.apache.zookeeper.data.ACL;
@@ -34,10 +37,12 @@ import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
+import java.nio.file.Paths;
import java.security.NoSuchAlgorithmException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
+import java.util.Properties;
public class TestZkConfigSetService extends SolrTestCaseJ4 {
@@ -205,6 +210,25 @@ public class TestZkConfigSetService extends SolrTestCaseJ4
{
}
+ @Test
+ public void testBootstrapConf() throws IOException, KeeperException,
InterruptedException {
+
+ String solrHome = SolrJettyTestBase.legacyExampleCollection1SolrHome();
+
+ CoreContainer cc = new CoreContainer(Paths.get(solrHome), new
Properties());
+ System.setProperty("zkHost", zkServer.getZkAddress());
+
+ SolrZkClient zkClient = new SolrZkClient(zkServer.getZkHost(),
AbstractZkTestCase.TIMEOUT);
+ zkClient.makePath("/solr", false, true);
+ cc.setCoreConfigService(new ZkConfigSetService(zkClient));
+ assertFalse(cc.getConfigSetService().checkConfigExists("collection1"));
+ ConfigSetService.bootstrapConf(cc);
+ assertTrue(cc.getConfigSetService().checkConfigExists("collection1"));
+
+ zkClient.close();
+ }
+
+
static SolrZkClient buildZkClient(String zkAddress, final ZkACLProvider
aclProvider,
final ZkCredentialsProvider
credentialsProvider) {
return new SolrZkClient(zkAddress, 10000){
