This is an automated email from the ASF dual-hosted git repository.
cpoerschke pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/solr.git
The following commit(s) were added to refs/heads/main by this push:
new ce9930338d2 SOLR-17050: Use compact JSON for Learning to Rank (LTR)
feature and model storage. (#2030)
ce9930338d2 is described below
commit ce9930338d2d576a3f83fc17754366c0bc0ca845
Author: Florin Babes <[email protected]>
AuthorDate: Mon Dec 4 16:21:05 2023 +0200
SOLR-17050: Use compact JSON for Learning to Rank (LTR) feature and model
storage. (#2030)
Co-authored-by: Florin Babes <[email protected]>
Co-authored-by: Christine Poerschke <[email protected]>
---
solr/CHANGES.txt | 2 ++
.../apache/solr/rest/ManagedResourceStorage.java | 14 +++++++-
.../src/test/org/apache/solr/util/TestUtils.java | 40 ++++++++++++++++++++++
.../solr/ltr/store/rest/ManagedFeatureStore.java | 6 ++++
.../solr/ltr/store/rest/ManagedModelStore.java | 6 ++++
.../store/rest/TestModelManagerPersistence.java | 20 +++++++++++
.../java/org/apache/solr/common/util/Utils.java | 21 +++++++++++-
solr/solrj/src/java/org/noggit/JSONWriter.java | 4 ++-
8 files changed, 110 insertions(+), 3 deletions(-)
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index da916a6be6c..9e63d1ce42e 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -113,6 +113,8 @@ Improvements
* SOLR-16959: Make the internal CoreSorter implementation configurable in
solr.xml (Vincent Primault)
+* SOLR-17050: Use compact JSON for Learning to Rank (LTR) feature and model
storage. (Florin Babes, Christine Poerschke, Alessandro Benedetti)
+
Optimizations
---------------------
* SOLR-17084: LBSolrClient (used by CloudSolrClient) now returns the count of
core tracked as not live AKA zombies
diff --git
a/solr/core/src/java/org/apache/solr/rest/ManagedResourceStorage.java
b/solr/core/src/java/org/apache/solr/rest/ManagedResourceStorage.java
index b28f4f57c36..af04c36eaab 100644
--- a/solr/core/src/java/org/apache/solr/rest/ManagedResourceStorage.java
+++ b/solr/core/src/java/org/apache/solr/rest/ManagedResourceStorage.java
@@ -47,6 +47,7 @@ import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.Utils;
import org.apache.solr.core.SolrResourceLoader;
+import org.noggit.JSONWriter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -420,9 +421,20 @@ public abstract class ManagedResourceStorage {
/** Default storage implementation that uses JSON as the storage format for
managed data. */
public static class JsonStorage extends ManagedResourceStorage {
+ private final int indentSize;
+ /** Uses {@link JSONWriter#DEFAULT_INDENT} space characters as an indent.
*/
public JsonStorage(StorageIO storageIO, SolrResourceLoader loader) {
+ this(storageIO, loader, JSONWriter.DEFAULT_INDENT);
+ }
+
+ /**
+ * @param indentSize The number of space characters to use as an indent.
0=newlines but no
+ * spaces, -1=no indent at all.
+ */
+ public JsonStorage(StorageIO storageIO, SolrResourceLoader loader, int
indentSize) {
super(storageIO, loader);
+ this.indentSize = indentSize;
}
/**
@@ -441,7 +453,7 @@ public abstract class ManagedResourceStorage {
@Override
public void store(String resourceId, Object toStore) throws IOException {
- String json = toJSONString(toStore);
+ String json = toJSONString(toStore, indentSize);
String storedResourceId = getStoredResourceId(resourceId);
OutputStreamWriter writer = null;
try {
diff --git a/solr/core/src/test/org/apache/solr/util/TestUtils.java
b/solr/core/src/test/org/apache/solr/util/TestUtils.java
index e05f3130360..3e18f8e766c 100644
--- a/solr/core/src/test/org/apache/solr/util/TestUtils.java
+++ b/solr/core/src/test/org/apache/solr/util/TestUtils.java
@@ -294,4 +294,44 @@ public class TestUtils extends SolrTestCaseJ4 {
assertEquals(
2L, Utils.getObjectByPath(sink, true, List.of(DEFAULTS,
COLLECTION_PROP, NRT_REPLICAS)));
}
+
+ @SuppressWarnings({"unchecked"})
+ public void testToJson() {
+ Map<String, Object> object =
+ (Map<String, Object>) Utils.fromJSONString("{k2:v2, k1: {a:b, p:r,
k21:{xx:yy}}}");
+
+ assertEquals(
+ "{\n"
+ + " \"k2\":\"v2\",\n"
+ + " \"k1\":{\n"
+ + " \"a\":\"b\",\n"
+ + " \"p\":\"r\",\n"
+ + " \"k21\":{\"xx\":\"yy\"}}}",
+ new String(Utils.toJSON(object), UTF_8));
+ }
+
+ @SuppressWarnings({"unchecked"})
+ public void testToJsonCompacted() {
+ Map<String, Object> object =
+ (Map<String, Object>) Utils.fromJSONString("{k2:v2, k1: {a:b, p:r,
k21:{xx:yy}}}");
+
+ assertEquals(
+
"{\"k2\":\"v2\",\"k1\":{\"a\":\"b\",\"p\":\"r\",\"k21\":{\"xx\":\"yy\"}}}",
+ new String(Utils.toJSON(object, -1), UTF_8));
+ }
+
+ @SuppressWarnings({"unchecked"})
+ public void testToJsonNoIndent() {
+ Map<String, Object> object =
+ (Map<String, Object>) Utils.fromJSONString("{k2:v2, k1: {a:b, p:r,
k21:{xx:yy}}}");
+
+ assertEquals(
+ "{\n"
+ + "\"k2\":\"v2\",\n"
+ + "\"k1\":{\n"
+ + "\"a\":\"b\",\n"
+ + "\"p\":\"r\",\n"
+ + "\"k21\":{\"xx\":\"yy\"}}}",
+ new String(Utils.toJSON(object, 0), UTF_8));
+ }
}
diff --git
a/solr/modules/ltr/src/java/org/apache/solr/ltr/store/rest/ManagedFeatureStore.java
b/solr/modules/ltr/src/java/org/apache/solr/ltr/store/rest/ManagedFeatureStore.java
index e867e7bbbad..96122f1604c 100644
---
a/solr/modules/ltr/src/java/org/apache/solr/ltr/store/rest/ManagedFeatureStore.java
+++
b/solr/modules/ltr/src/java/org/apache/solr/ltr/store/rest/ManagedFeatureStore.java
@@ -85,6 +85,12 @@ public class ManagedFeatureStore extends ManagedResource
super(resourceId, loader, storageIO);
}
+ @Override
+ protected ManagedResourceStorage createStorage(
+ ManagedResourceStorage.StorageIO storageIO, SolrResourceLoader loader)
throws SolrException {
+ return new ManagedResourceStorage.JsonStorage(storageIO, loader, -1);
+ }
+
public Map<String, FeatureStore> getStores() {
return stores;
}
diff --git
a/solr/modules/ltr/src/java/org/apache/solr/ltr/store/rest/ManagedModelStore.java
b/solr/modules/ltr/src/java/org/apache/solr/ltr/store/rest/ManagedModelStore.java
index 918b06bd724..f6778e54425 100644
---
a/solr/modules/ltr/src/java/org/apache/solr/ltr/store/rest/ManagedModelStore.java
+++
b/solr/modules/ltr/src/java/org/apache/solr/ltr/store/rest/ManagedModelStore.java
@@ -93,6 +93,12 @@ public class ManagedModelStore extends ManagedResource
store = new ModelStore();
}
+ @Override
+ protected ManagedResourceStorage createStorage(
+ ManagedResourceStorage.StorageIO storageIO, SolrResourceLoader loader)
throws SolrException {
+ return new ManagedResourceStorage.JsonStorage(storageIO, loader, -1);
+ }
+
public void setManagedFeatureStore(ManagedFeatureStore managedFeatureStore) {
log.info("INIT model store");
this.managedFeatureStore = managedFeatureStore;
diff --git
a/solr/modules/ltr/src/test/org/apache/solr/ltr/store/rest/TestModelManagerPersistence.java
b/solr/modules/ltr/src/test/org/apache/solr/ltr/store/rest/TestModelManagerPersistence.java
index 2adcc1f19f8..7b159e0e650 100644
---
a/solr/modules/ltr/src/test/org/apache/solr/ltr/store/rest/TestModelManagerPersistence.java
+++
b/solr/modules/ltr/src/test/org/apache/solr/ltr/store/rest/TestModelManagerPersistence.java
@@ -16,6 +16,8 @@
*/
package org.apache.solr.ltr.store.rest;
+import static java.nio.charset.StandardCharsets.UTF_8;
+
import java.io.BufferedWriter;
import java.io.OutputStreamWriter;
import java.nio.charset.StandardCharsets;
@@ -137,6 +139,24 @@ public class TestModelManagerPersistence extends
TestRerankBase {
assertJQ(ManagedModelStore.REST_END_POINT, "/models/==[]");
}
+ @Test
+ public void testFeaturesAndModelAreStoredCompact() throws Exception {
+ loadFeature("feature", ValueFeature.class.getName(), "test",
"{\"value\":2}");
+ loadModel(
+ "test-model",
+ LinearModel.class.getName(),
+ new String[] {"feature"},
+ "test",
+ "{\"weights\":{\"feature\":1.0}}");
+
+ final String fstorecontent = Files.readString(fstorefile,
StandardCharsets.UTF_8);
+ final String mstorecontent = Files.readString(mstorefile,
StandardCharsets.UTF_8);
+ Object fStoreObject = Utils.fromJSONString(fstorecontent);
+ Object mStoreObject = Utils.fromJSONString(mstorecontent);
+ assertEquals(new String(Utils.toJSON(fStoreObject, -1), UTF_8),
fstorecontent);
+ assertEquals(new String(Utils.toJSON(mStoreObject, -1), UTF_8),
mstorecontent);
+ }
+
@Test
public void testFilePersistence() throws Exception {
// check whether models and features are empty
diff --git a/solr/solrj/src/java/org/apache/solr/common/util/Utils.java
b/solr/solrj/src/java/org/apache/solr/common/util/Utils.java
index 0ae4886d670..2a3ea47b103 100644
--- a/solr/solrj/src/java/org/apache/solr/common/util/Utils.java
+++ b/solr/solrj/src/java/org/apache/solr/common/util/Utils.java
@@ -217,10 +217,29 @@ public class Utils {
public static byte[] toJSON(Object o) {
if (o == null) return new byte[0];
CharArr out = new CharArr();
- new JSONWriter(out, 2).write(o); // indentation by default
+ new JSONWriter(out, JSONWriter.DEFAULT_INDENT).write(o); // indentation by
default
return toUTF8(out);
}
+ /**
+ * @param indentSize The number of space characters to use as an indent.
0=newlines but no spaces,
+ * -1=no indent at all.
+ */
+ public static byte[] toJSON(Object o, int indentSize) {
+ if (o == null) return new byte[0];
+ CharArr out = new CharArr();
+ new JSONWriter(out, indentSize).write(o);
+ return toUTF8(out);
+ }
+
+ /**
+ * @param indentSize The number of space characters to use as an indent.
0=newlines but no spaces,
+ * -1=no indent at all.
+ */
+ public static String toJSONString(Object o, int indentSize) {
+ return new String(toJSON(o, indentSize), StandardCharsets.UTF_8);
+ }
+
public static String toJSONString(Object o) {
return new String(toJSON(o), StandardCharsets.UTF_8);
}
diff --git a/solr/solrj/src/java/org/noggit/JSONWriter.java
b/solr/solrj/src/java/org/noggit/JSONWriter.java
index 349c1fecf02..9b1dd073c83 100644
--- a/solr/solrj/src/java/org/noggit/JSONWriter.java
+++ b/solr/solrj/src/java/org/noggit/JSONWriter.java
@@ -31,6 +31,8 @@ public class JSONWriter {
public void write(JSONWriter writer);
}
+ public static final int DEFAULT_INDENT = 2;
+
protected int level;
protected int indent;
protected final CharArr out;
@@ -46,7 +48,7 @@ public class JSONWriter {
}
public JSONWriter(CharArr out) {
- this(out, 2);
+ this(out, DEFAULT_INDENT);
}
public void setIndentSize(int indentSize) {
