This is an automated email from the ASF dual-hosted git repository.
jsweeney pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/solr.git
The following commit(s) were added to refs/heads/main by this push:
new 0e7a05de565 Adding support for compression within the ZkCLI tool
(#1348)
0e7a05de565 is described below
commit 0e7a05de5656e4d686eedde00863d88d4d820410
Author: Justin Sweeney <[email protected]>
AuthorDate: Tue Feb 28 07:52:17 2023 -0700
Adding support for compression within the ZkCLI tool (#1348)
This provides support for using the ZkCLI tool to get/put compressed
state.json data, providing the ability to easily edit state.json even when
compression is enabled.
---
.../core/src/java/org/apache/solr/cloud/ZkCLI.java | 80 +++++++++++++--
.../src/test/org/apache/solr/cloud/ZkCLITest.java | 108 +++++++++++++++++++++
solr/server/scripts/cloud-scripts/zkcli.sh | 4 +-
3 files changed, 181 insertions(+), 11 deletions(-)
diff --git a/solr/core/src/java/org/apache/solr/cloud/ZkCLI.java
b/solr/core/src/java/org/apache/solr/cloud/ZkCLI.java
index f8ae0e194de..9cac8e45661 100644
--- a/solr/core/src/java/org/apache/solr/cloud/ZkCLI.java
+++ b/solr/core/src/java/org/apache/solr/cloud/ZkCLI.java
@@ -25,7 +25,9 @@ import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.PrintStream;
+import java.lang.reflect.InvocationTargetException;
import java.nio.charset.StandardCharsets;
+import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.List;
import java.util.Properties;
@@ -42,11 +44,17 @@ import org.apache.commons.cli.ParseException;
import org.apache.commons.cli.PosixParser;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.StringUtils;
import org.apache.solr.common.cloud.ClusterProperties;
import org.apache.solr.common.cloud.SolrZkClient;
import org.apache.solr.common.cloud.ZkMaintenanceUtils;
+import org.apache.solr.common.util.Compressor;
+import org.apache.solr.common.util.ZLibCompressor;
import org.apache.solr.core.ConfigSetService;
import org.apache.solr.core.CoreContainer;
+import org.apache.solr.core.NodeConfig;
+import org.apache.solr.core.SolrXmlConfig;
import org.apache.solr.util.CLIO;
import org.apache.zookeeper.CreateMode;
import org.apache.zookeeper.KeeperException;
@@ -234,6 +242,9 @@ public class ZkCLI implements CLIO {
// start up a tmp zk server first
String zkServerAddress = line.getOptionValue(ZKHOST);
String solrHome = line.getOptionValue(SOLRHOME);
+ if (StringUtils.isEmpty(solrHome)) {
+ solrHome = System.getProperty("solr.home");
+ }
String solrPort = null;
if (line.hasOption(RUNZK)) {
@@ -256,6 +267,38 @@ public class ZkCLI implements CLIO {
zkServer.parseConfig();
zkServer.start();
}
+
+ int minStateByteLenForCompression = -1;
+ Compressor compressor = new ZLibCompressor();
+
+ if (solrHome != null) {
+ try {
+ Path solrHomePath = Paths.get(solrHome);
+ Properties props = new Properties();
+ props.put(SolrXmlConfig.ZK_HOST, zkServerAddress);
+ NodeConfig nodeConfig = NodeConfig.loadNodeConfig(solrHomePath,
props);
+ minStateByteLenForCompression =
+ nodeConfig.getCloudConfig().getMinStateByteLenForCompression();
+ String stateCompressorClass =
nodeConfig.getCloudConfig().getStateCompressorClass();
+ if (!StringUtils.isEmpty(stateCompressorClass)) {
+ Class<? extends Compressor> compressionClass =
+
Class.forName(stateCompressorClass).asSubclass(Compressor.class);
+ compressor =
compressionClass.getDeclaredConstructor().newInstance();
+ }
+ } catch (SolrException e) {
+ // Failed to load solr.xml
+ stdout.println(
+ "Failed to load solr.xml from ZK or SolrHome, put/get operations
on compressed data will use data as is. If you intention is to read and
de-compress data or compress and write data, then solr.xml must be
accessible.");
+ } catch (ClassNotFoundException
+ | NoSuchMethodException
+ | InstantiationException
+ | IllegalAccessException
+ | InvocationTargetException e) {
+ stdout.println("Unable to find or instantiate compression class: " +
e.getMessage());
+ System.exit(1);
+ }
+ }
+
SolrZkClient zkClient = null;
try {
zkClient =
@@ -264,6 +307,7 @@ public class ZkCLI implements CLIO {
.withTimeout(30000, TimeUnit.MILLISECONDS)
.withConnTimeOut(30000, TimeUnit.MILLISECONDS)
.withReconnectListener(() -> {})
+ .withCompressor(compressor)
.build();
if (line.getOptionValue(CMD).equalsIgnoreCase(BOOTSTRAP)) {
@@ -359,16 +403,16 @@ public class ZkCLI implements CLIO {
"-" + PUT + " requires two args - the path to create and the
data string");
System.exit(1);
}
- String path = arglist.get(0).toString();
+ String path = arglist.get(0);
+ byte[] data = arglist.get(1).getBytes(StandardCharsets.UTF_8);
+ if (shouldCompressData(data, path, minStateByteLenForCompression)) {
+ // state.json should be compressed before being put to ZK
+ data = compressor.compressBytes(data);
+ }
if (zkClient.exists(path, true)) {
- zkClient.setData(
- path,
arglist.get(1).toString().getBytes(StandardCharsets.UTF_8), true);
+ zkClient.setData(path, data, true);
} else {
- zkClient.create(
- path,
- arglist.get(1).toString().getBytes(StandardCharsets.UTF_8),
- CreateMode.PERSISTENT,
- true);
+ zkClient.create(path, data, CreateMode.PERSISTENT, true);
}
} else if (line.getOptionValue(CMD).equalsIgnoreCase(PUT_FILE)) {
List<String> arglist = line.getArgList();
@@ -382,11 +426,16 @@ public class ZkCLI implements CLIO {
String path = arglist.get(0).toString();
InputStream is = new FileInputStream(arglist.get(1).toString());
+ byte[] data = IOUtils.toByteArray(is);
+ if (shouldCompressData(data, path, minStateByteLenForCompression)) {
+ // state.json should be compressed before being put to ZK
+ data = compressor.compressBytes(data);
+ }
try {
if (zkClient.exists(path, true)) {
- zkClient.setData(path, IOUtils.toByteArray(is), true);
+ zkClient.setData(path, data, true);
} else {
- zkClient.create(path, IOUtils.toByteArray(is),
CreateMode.PERSISTENT, true);
+ zkClient.create(path, data, CreateMode.PERSISTENT, true);
}
} finally {
IOUtils.closeQuietly(is);
@@ -450,4 +499,15 @@ public class ZkCLI implements CLIO {
stdout.println("Unexpected exception:" + exp.getMessage());
}
}
+
+ private static boolean shouldCompressData(
+ byte[] data, String path, int minStateByteLenForCompression) {
+ if (path.endsWith("state.json")
+ && minStateByteLenForCompression > -1
+ && data.length > minStateByteLenForCompression) {
+ // state.json should be compressed before being put to ZK
+ return true;
+ }
+ return false;
+ }
}
diff --git a/solr/core/src/test/org/apache/solr/cloud/ZkCLITest.java
b/solr/core/src/test/org/apache/solr/cloud/ZkCLITest.java
index 58101f0c7c6..1ba4fde8caa 100644
--- a/solr/core/src/test/org/apache/solr/cloud/ZkCLITest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/ZkCLITest.java
@@ -32,6 +32,7 @@ import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
import org.apache.commons.io.filefilter.RegexFileFilter;
import org.apache.commons.io.filefilter.TrueFileFilter;
+import org.apache.commons.lang3.StringUtils;
import org.apache.solr.SolrJettyTestBase;
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.common.SolrException;
@@ -40,6 +41,7 @@ import org.apache.solr.common.cloud.SolrZkClient;
import org.apache.solr.common.cloud.VMParamsAllAndReadonlyDigestZkACLProvider;
import org.apache.solr.common.cloud.ZkNodeProps;
import org.apache.solr.common.cloud.ZkStateReader;
+import org.apache.solr.common.util.ZLibCompressor;
import org.apache.solr.core.ConfigSetService;
import org.apache.solr.util.ExternalPaths;
import org.apache.zookeeper.CreateMode;
@@ -63,6 +65,8 @@ public class ZkCLITest extends SolrTestCaseJ4 {
private SolrZkClient zkClient;
+ private PrintStream originalSystemOut;
+
protected static final String SOLR_HOME = SolrTestCaseJ4.TEST_HOME();
@BeforeClass
@@ -87,6 +91,8 @@ public class ZkCLITest extends SolrTestCaseJ4 {
Path tmpDir = createTempDir();
solrHome = exampleHome;
+ originalSystemOut = System.out;
+
zkDir = tmpDir.resolve("zookeeper/server1/data");
log.info("ZooKeeper dataDir:{}", zkDir);
zkServer = new ZkTestServer(zkDir);
@@ -170,6 +176,28 @@ public class ZkCLITest extends SolrTestCaseJ4 {
zkClient.getData("/data.txt", null, null, true),
data.getBytes(StandardCharsets.UTF_8));
}
+ @Test
+ public void testPutCompressed() throws Exception {
+ // test put compressed
+ System.setProperty("solr.home", solrHome);
+ System.setProperty("minStateByteLenForCompression", "0");
+
+ String data = "my data";
+ ZLibCompressor zLibCompressor = new ZLibCompressor();
+ byte[] expected =
zLibCompressor.compressBytes(data.getBytes(StandardCharsets.UTF_8));
+ String[] args =
+ new String[] {"-zkhost", zkServer.getZkAddress(), "-cmd", "put",
"/state.json", data};
+ ZkCLI.main(args);
+ assertArrayEquals(zkClient.getZooKeeper().getData("/state.json", null,
null), expected);
+
+ // test re-put to existing
+ data = "my data deux";
+ expected =
zLibCompressor.compressBytes(data.getBytes(StandardCharsets.UTF_8));
+ args = new String[] {"-zkhost", zkServer.getZkAddress(), "-cmd", "put",
"/state.json", data};
+ ZkCLI.main(args);
+ assertArrayEquals(zkClient.getZooKeeper().getData("/state.json", null,
null), expected);
+ }
+
@Test
public void testPutFile() throws Exception {
// test put file
@@ -197,6 +225,37 @@ public class ZkCLITest extends SolrTestCaseJ4 {
assertEquals("Should get back what we put in ZK", fromZk, fromLoc);
}
+ @Test
+ public void testPutFileCompressed() throws Exception {
+ // test put file compressed
+ System.setProperty("solr.home", solrHome);
+ System.setProperty("minStateByteLenForCompression", "0");
+
+ String[] args =
+ new String[] {
+ "-zkhost",
+ zkServer.getZkAddress(),
+ "-cmd",
+ "putfile",
+ "/state.json",
+ SOLR_HOME + File.separator + "solr-stress-new.xml"
+ };
+ ZkCLI.main(args);
+
+ byte[] fromZk = zkClient.getZooKeeper().getData("/state.json", null, null);
+ File locFile = new File(SOLR_HOME + File.separator +
"solr-stress-new.xml");
+ InputStream is = new FileInputStream(locFile);
+ byte[] fromLoc;
+ try {
+ fromLoc = IOUtils.toByteArray(is);
+ ZLibCompressor zLibCompressor = new ZLibCompressor();
+ fromLoc = zLibCompressor.compressBytes(fromLoc);
+ } finally {
+ IOUtils.closeQuietly(is);
+ }
+ assertArrayEquals("Should get back what we put in ZK", fromLoc, fromZk);
+ }
+
@Test
public void testPutFileNotExists() {
// test put file
@@ -379,9 +438,34 @@ public class ZkCLITest extends SolrTestCaseJ4 {
public void testGet() throws Exception {
String getNode = "/getNode";
byte[] data = "getNode-data".getBytes(StandardCharsets.UTF_8);
+ ByteArrayOutputStream systemOut = new ByteArrayOutputStream();
this.zkClient.create(getNode, data, CreateMode.PERSISTENT, true);
String[] args = new String[] {"-zkhost", zkServer.getZkAddress(), "-cmd",
"get", getNode};
+ ZkCLI.setStdout(new PrintStream(systemOut, true, StandardCharsets.UTF_8));
ZkCLI.main(args);
+ assertArrayEquals(
+ data,
+ StringUtils.removeEnd(systemOut.toString(StandardCharsets.UTF_8), "\n")
+ .getBytes(StandardCharsets.UTF_8));
+ }
+
+ @Test
+ public void testGetCompressed() throws Exception {
+ System.setProperty("solr.home", solrHome);
+ System.setProperty("minStateByteLenForCompression", "0");
+
+ String getNode = "/getNode";
+ byte[] data = "getNode-data".getBytes(StandardCharsets.UTF_8);
+ ZLibCompressor zLibCompressor = new ZLibCompressor();
+ ByteArrayOutputStream systemOut = new ByteArrayOutputStream();
+ this.zkClient.create(getNode, zLibCompressor.compressBytes(data),
CreateMode.PERSISTENT, true);
+ String[] args = new String[] {"-zkhost", zkServer.getZkAddress(), "-cmd",
"get", getNode};
+ ZkCLI.setStdout(new PrintStream(systemOut, true, StandardCharsets.UTF_8));
+ ZkCLI.main(args);
+ assertArrayEquals(
+ data,
+ StringUtils.removeEnd(systemOut.toString(StandardCharsets.UTF_8), "\n")
+ .getBytes(StandardCharsets.UTF_8));
}
@Test
@@ -404,6 +488,27 @@ public class ZkCLITest extends SolrTestCaseJ4 {
assertArrayEquals(data, readData);
}
+ @Test
+ public void testGetFileCompressed() throws Exception {
+ File tmpDir = createTempDir().toFile();
+
+ String getNode = "/getFileNode";
+ byte[] data = "getFileNode-data".getBytes(StandardCharsets.UTF_8);
+ ZLibCompressor zLibCompressor = new ZLibCompressor();
+ this.zkClient.create(getNode, zLibCompressor.compressBytes(data),
CreateMode.PERSISTENT, true);
+
+ File file =
+ new File(tmpDir, "solrtest-getfile-" + this.getClass().getName() + "-"
+ System.nanoTime());
+ String[] args =
+ new String[] {
+ "-zkhost", zkServer.getZkAddress(), "-cmd", "getfile", getNode,
file.getAbsolutePath()
+ };
+ ZkCLI.main(args);
+
+ byte[] readData = FileUtils.readFileToByteArray(file);
+ assertArrayEquals(data, readData);
+ }
+
@Test
public void testGetFileNotExists() throws Exception {
String getNode = "/getFileNotExistsNode";
@@ -502,6 +607,9 @@ public class ZkCLITest extends SolrTestCaseJ4 {
if (zkServer != null) {
zkServer.shutdown();
}
+ System.clearProperty("solr.home");
+ System.clearProperty("minStateByteLenForCompression");
+ System.setOut(originalSystemOut);
super.tearDown();
}
}
diff --git a/solr/server/scripts/cloud-scripts/zkcli.sh
b/solr/server/scripts/cloud-scripts/zkcli.sh
index fa4e4556bfa..4477df48634 100755
--- a/solr/server/scripts/cloud-scripts/zkcli.sh
+++ b/solr/server/scripts/cloud-scripts/zkcli.sh
@@ -11,6 +11,8 @@ sdir="`dirname \"$0\"`"
log4j_config="file:$sdir/../../resources/log4j2-console.xml"
+solr_home="$sdir/../../solr"
+
# Settings for ZK ACL
#SOLR_ZK_CREDS_AND_ACLS="-DzkACLProvider=org.apache.solr.common.cloud.DigestZkACLProvider
\
#
-DzkCredentialsProvider=org.apache.solr.common.cloud.DigestZkCredentialsProvider
\
@@ -21,6 +23,6 @@ log4j_config="file:$sdir/../../resources/log4j2-console.xml"
#...
# -DzkDigestCredentialsFile=/path/to/zkDigestCredentialsFile.properties
#...
-PATH=$JAVA_HOME/bin:$PATH $JVM $SOLR_ZK_CREDS_AND_ACLS $ZKCLI_JVM_FLAGS
-Dlog4j.configurationFile=$log4j_config \
+PATH=$JAVA_HOME/bin:$PATH $JVM $SOLR_ZK_CREDS_AND_ACLS $ZKCLI_JVM_FLAGS
-Dlog4j.configurationFile=$log4j_config -DsolrHome=$solr_home \
-classpath
"$sdir/../../solr-webapp/webapp/WEB-INF/lib/*:$sdir/../../lib/ext/*:$sdir/../../lib/*"
org.apache.solr.cloud.ZkCLI ${1+"$@"}
