This is an automated email from the ASF dual-hosted git repository. cpoerschke pushed a commit to branch branch_9x in repository https://gitbox.apache.org/repos/asf/solr.git
commit 2ff81a7cb6b9344baf5f4b050b8a72b2777dba19 Author: Florin Babes <[email protected]> AuthorDate: Mon Dec 4 16:21:05 2023 +0200 SOLR-17050: Use compact JSON for Learning to Rank (LTR) feature and model storage. (#2030) Co-authored-by: Florin Babes <[email protected]> Co-authored-by: Christine Poerschke <[email protected]> (cherry picked from commit ce9930338d2d576a3f83fc17754366c0bc0ca845) --- solr/CHANGES.txt | 2 ++ .../apache/solr/rest/ManagedResourceStorage.java | 14 +++++++- .../src/test/org/apache/solr/util/TestUtils.java | 40 ++++++++++++++++++++++ .../solr/ltr/store/rest/ManagedFeatureStore.java | 6 ++++ .../solr/ltr/store/rest/ManagedModelStore.java | 6 ++++ .../store/rest/TestModelManagerPersistence.java | 20 +++++++++++ .../java/org/apache/solr/common/util/Utils.java | 21 +++++++++++- solr/solrj/src/java/org/noggit/JSONWriter.java | 4 ++- 8 files changed, 110 insertions(+), 3 deletions(-) diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index cabde3e61bd..95eb594c31b 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -45,6 +45,8 @@ Improvements * SOLR-16959: Make the internal CoreSorter implementation configurable in solr.xml (Vincent Primault) +* SOLR-17050: Use compact JSON for Learning to Rank (LTR) feature and model storage. (Florin Babes, Christine Poerschke, Alessandro Benedetti) + Optimizations --------------------- * SOLR-17084: LBSolrClient (used by CloudSolrClient) now returns the count of core tracked as not live AKA zombies diff --git a/solr/core/src/java/org/apache/solr/rest/ManagedResourceStorage.java b/solr/core/src/java/org/apache/solr/rest/ManagedResourceStorage.java index b28f4f57c36..af04c36eaab 100644 --- a/solr/core/src/java/org/apache/solr/rest/ManagedResourceStorage.java +++ b/solr/core/src/java/org/apache/solr/rest/ManagedResourceStorage.java @@ -47,6 +47,7 @@ import org.apache.solr.common.cloud.ZkStateReader; import org.apache.solr.common.util.NamedList; import org.apache.solr.common.util.Utils; import org.apache.solr.core.SolrResourceLoader; +import org.noggit.JSONWriter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -420,9 +421,20 @@ public abstract class ManagedResourceStorage { /** Default storage implementation that uses JSON as the storage format for managed data. */ public static class JsonStorage extends ManagedResourceStorage { + private final int indentSize; + /** Uses {@link JSONWriter#DEFAULT_INDENT} space characters as an indent. */ public JsonStorage(StorageIO storageIO, SolrResourceLoader loader) { + this(storageIO, loader, JSONWriter.DEFAULT_INDENT); + } + + /** + * @param indentSize The number of space characters to use as an indent. 0=newlines but no + * spaces, -1=no indent at all. + */ + public JsonStorage(StorageIO storageIO, SolrResourceLoader loader, int indentSize) { super(storageIO, loader); + this.indentSize = indentSize; } /** @@ -441,7 +453,7 @@ public abstract class ManagedResourceStorage { @Override public void store(String resourceId, Object toStore) throws IOException { - String json = toJSONString(toStore); + String json = toJSONString(toStore, indentSize); String storedResourceId = getStoredResourceId(resourceId); OutputStreamWriter writer = null; try { diff --git a/solr/core/src/test/org/apache/solr/util/TestUtils.java b/solr/core/src/test/org/apache/solr/util/TestUtils.java index e05f3130360..3e18f8e766c 100644 --- a/solr/core/src/test/org/apache/solr/util/TestUtils.java +++ b/solr/core/src/test/org/apache/solr/util/TestUtils.java @@ -294,4 +294,44 @@ public class TestUtils extends SolrTestCaseJ4 { assertEquals( 2L, Utils.getObjectByPath(sink, true, List.of(DEFAULTS, COLLECTION_PROP, NRT_REPLICAS))); } + + @SuppressWarnings({"unchecked"}) + public void testToJson() { + Map<String, Object> object = + (Map<String, Object>) Utils.fromJSONString("{k2:v2, k1: {a:b, p:r, k21:{xx:yy}}}"); + + assertEquals( + "{\n" + + " \"k2\":\"v2\",\n" + + " \"k1\":{\n" + + " \"a\":\"b\",\n" + + " \"p\":\"r\",\n" + + " \"k21\":{\"xx\":\"yy\"}}}", + new String(Utils.toJSON(object), UTF_8)); + } + + @SuppressWarnings({"unchecked"}) + public void testToJsonCompacted() { + Map<String, Object> object = + (Map<String, Object>) Utils.fromJSONString("{k2:v2, k1: {a:b, p:r, k21:{xx:yy}}}"); + + assertEquals( + "{\"k2\":\"v2\",\"k1\":{\"a\":\"b\",\"p\":\"r\",\"k21\":{\"xx\":\"yy\"}}}", + new String(Utils.toJSON(object, -1), UTF_8)); + } + + @SuppressWarnings({"unchecked"}) + public void testToJsonNoIndent() { + Map<String, Object> object = + (Map<String, Object>) Utils.fromJSONString("{k2:v2, k1: {a:b, p:r, k21:{xx:yy}}}"); + + assertEquals( + "{\n" + + "\"k2\":\"v2\",\n" + + "\"k1\":{\n" + + "\"a\":\"b\",\n" + + "\"p\":\"r\",\n" + + "\"k21\":{\"xx\":\"yy\"}}}", + new String(Utils.toJSON(object, 0), UTF_8)); + } } diff --git a/solr/modules/ltr/src/java/org/apache/solr/ltr/store/rest/ManagedFeatureStore.java b/solr/modules/ltr/src/java/org/apache/solr/ltr/store/rest/ManagedFeatureStore.java index e867e7bbbad..96122f1604c 100644 --- a/solr/modules/ltr/src/java/org/apache/solr/ltr/store/rest/ManagedFeatureStore.java +++ b/solr/modules/ltr/src/java/org/apache/solr/ltr/store/rest/ManagedFeatureStore.java @@ -85,6 +85,12 @@ public class ManagedFeatureStore extends ManagedResource super(resourceId, loader, storageIO); } + @Override + protected ManagedResourceStorage createStorage( + ManagedResourceStorage.StorageIO storageIO, SolrResourceLoader loader) throws SolrException { + return new ManagedResourceStorage.JsonStorage(storageIO, loader, -1); + } + public Map<String, FeatureStore> getStores() { return stores; } diff --git a/solr/modules/ltr/src/java/org/apache/solr/ltr/store/rest/ManagedModelStore.java b/solr/modules/ltr/src/java/org/apache/solr/ltr/store/rest/ManagedModelStore.java index 918b06bd724..f6778e54425 100644 --- a/solr/modules/ltr/src/java/org/apache/solr/ltr/store/rest/ManagedModelStore.java +++ b/solr/modules/ltr/src/java/org/apache/solr/ltr/store/rest/ManagedModelStore.java @@ -93,6 +93,12 @@ public class ManagedModelStore extends ManagedResource store = new ModelStore(); } + @Override + protected ManagedResourceStorage createStorage( + ManagedResourceStorage.StorageIO storageIO, SolrResourceLoader loader) throws SolrException { + return new ManagedResourceStorage.JsonStorage(storageIO, loader, -1); + } + public void setManagedFeatureStore(ManagedFeatureStore managedFeatureStore) { log.info("INIT model store"); this.managedFeatureStore = managedFeatureStore; diff --git a/solr/modules/ltr/src/test/org/apache/solr/ltr/store/rest/TestModelManagerPersistence.java b/solr/modules/ltr/src/test/org/apache/solr/ltr/store/rest/TestModelManagerPersistence.java index 2adcc1f19f8..7b159e0e650 100644 --- a/solr/modules/ltr/src/test/org/apache/solr/ltr/store/rest/TestModelManagerPersistence.java +++ b/solr/modules/ltr/src/test/org/apache/solr/ltr/store/rest/TestModelManagerPersistence.java @@ -16,6 +16,8 @@ */ package org.apache.solr.ltr.store.rest; +import static java.nio.charset.StandardCharsets.UTF_8; + import java.io.BufferedWriter; import java.io.OutputStreamWriter; import java.nio.charset.StandardCharsets; @@ -137,6 +139,24 @@ public class TestModelManagerPersistence extends TestRerankBase { assertJQ(ManagedModelStore.REST_END_POINT, "/models/==[]"); } + @Test + public void testFeaturesAndModelAreStoredCompact() throws Exception { + loadFeature("feature", ValueFeature.class.getName(), "test", "{\"value\":2}"); + loadModel( + "test-model", + LinearModel.class.getName(), + new String[] {"feature"}, + "test", + "{\"weights\":{\"feature\":1.0}}"); + + final String fstorecontent = Files.readString(fstorefile, StandardCharsets.UTF_8); + final String mstorecontent = Files.readString(mstorefile, StandardCharsets.UTF_8); + Object fStoreObject = Utils.fromJSONString(fstorecontent); + Object mStoreObject = Utils.fromJSONString(mstorecontent); + assertEquals(new String(Utils.toJSON(fStoreObject, -1), UTF_8), fstorecontent); + assertEquals(new String(Utils.toJSON(mStoreObject, -1), UTF_8), mstorecontent); + } + @Test public void testFilePersistence() throws Exception { // check whether models and features are empty diff --git a/solr/solrj/src/java/org/apache/solr/common/util/Utils.java b/solr/solrj/src/java/org/apache/solr/common/util/Utils.java index da30dbe19b7..9bb772ceac5 100644 --- a/solr/solrj/src/java/org/apache/solr/common/util/Utils.java +++ b/solr/solrj/src/java/org/apache/solr/common/util/Utils.java @@ -218,10 +218,29 @@ public class Utils { public static byte[] toJSON(Object o) { if (o == null) return new byte[0]; CharArr out = new CharArr(); - new JSONWriter(out, 2).write(o); // indentation by default + new JSONWriter(out, JSONWriter.DEFAULT_INDENT).write(o); // indentation by default return toUTF8(out); } + /** + * @param indentSize The number of space characters to use as an indent. 0=newlines but no spaces, + * -1=no indent at all. + */ + public static byte[] toJSON(Object o, int indentSize) { + if (o == null) return new byte[0]; + CharArr out = new CharArr(); + new JSONWriter(out, indentSize).write(o); + return toUTF8(out); + } + + /** + * @param indentSize The number of space characters to use as an indent. 0=newlines but no spaces, + * -1=no indent at all. + */ + public static String toJSONString(Object o, int indentSize) { + return new String(toJSON(o, indentSize), StandardCharsets.UTF_8); + } + public static String toJSONString(Object o) { return new String(toJSON(o), StandardCharsets.UTF_8); } diff --git a/solr/solrj/src/java/org/noggit/JSONWriter.java b/solr/solrj/src/java/org/noggit/JSONWriter.java index 349c1fecf02..9b1dd073c83 100644 --- a/solr/solrj/src/java/org/noggit/JSONWriter.java +++ b/solr/solrj/src/java/org/noggit/JSONWriter.java @@ -31,6 +31,8 @@ public class JSONWriter { public void write(JSONWriter writer); } + public static final int DEFAULT_INDENT = 2; + protected int level; protected int indent; protected final CharArr out; @@ -46,7 +48,7 @@ public class JSONWriter { } public JSONWriter(CharArr out) { - this(out, 2); + this(out, DEFAULT_INDENT); } public void setIndentSize(int indentSize) {
