This is an automated email from the ASF dual-hosted git repository.
magibney pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/solr.git
The following commit(s) were added to refs/heads/main by this push:
new 6ff8131 SOLR-9376: [xml] and [json] RawValue DocTransformers should
work in cloud mode (#513)
6ff8131 is described below
commit 6ff81312607dd5d33f87dc52aed9d52938dc6883
Author: Michael Gibney <[email protected]>
AuthorDate: Tue Jan 25 14:44:41 2022 -0500
SOLR-9376: [xml] and [json] RawValue DocTransformers should work in cloud
mode (#513)
---
solr/CHANGES.txt | 3 +
.../solr/response/GeoJSONResponseWriter.java | 7 +-
.../apache/solr/response/JSONResponseWriter.java | 14 +-
.../java/org/apache/solr/response/JSONWriter.java | 2 +-
.../apache/solr/response/PHPResponseWriter.java | 4 +-
.../solr/response/PHPSerializedResponseWriter.java | 8 +-
.../solr/response/RawShimTextResponseWriter.java | 111 +++++++
.../org/apache/solr/response/SchemaXmlWriter.java | 8 +-
.../solr/response/TabularResponseWriter.java | 2 +-
.../apache/solr/response/TextResponseWriter.java | 47 ++-
.../java/org/apache/solr/response/XMLWriter.java | 23 +-
.../solr/response/transform/DocTransformer.java | 18 ++
.../solr/response/transform/DocTransformers.java | 9 +
.../response/transform/GeoTransformerFactory.java | 148 +++++----
.../transform/RawValueTransformerFactory.java | 94 +++---
.../response/transform/RenameFieldTransformer.java | 11 +-
.../response/transform/TransformerFactory.java | 57 ++++
.../solr/response/transform/WriteableGeoJSON.java | 55 ----
.../org/apache/solr/search/SolrReturnFields.java | 101 ++++--
.../apache/solr/cloud/TestRandomFlRTGCloud.java | 351 ++++++++++++++++++---
.../org/apache/solr/response/JSONWriterTest.java | 4 +-
.../apache/solr/response/TestRawTransformer.java | 184 +++++++++--
.../handler/extraction/XLSXResponseWriter.java | 5 +-
.../src/major-changes-in-solr-9.adoc | 7 +
.../solr/client/solrj/impl/XMLResponseParser.java | 38 +++
.../apache/solr/common/util/JsonTextWriter.java | 16 +-
.../org/apache/solr/common/util/TextWriter.java | 55 +++-
.../apache/solr/common/util/WriteableValue.java | 25 --
28 files changed, 1060 insertions(+), 347 deletions(-)
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index fbef2ff..abbcdbc 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -266,6 +266,9 @@ when told to. The admin UI now tells it to. (Nazerke
Seidan, David Smiley)
* SOLR-15884: Backup responses now use a map to return information instead of
a list (Houston Putman, Christine Poerschke)
+* SOLR-9376: Raw value DocTransformers (`[xml]`, `[json]`, `[geo w=GeoJSON]`)
now work
+ in a distributed/SolrCloud context (Michael Gibney)
+
Build
---------------------
* LUCENE-9077 LUCENE-9433: Support Gradle build, remove Ant support from trunk
(Dawid Weiss, Erick Erickson, Uwe Schindler et.al.)
diff --git
a/solr/core/src/java/org/apache/solr/response/GeoJSONResponseWriter.java
b/solr/core/src/java/org/apache/solr/response/GeoJSONResponseWriter.java
index 89d556b..c16e5b6 100644
--- a/solr/core/src/java/org/apache/solr/response/GeoJSONResponseWriter.java
+++ b/solr/core/src/java/org/apache/solr/response/GeoJSONResponseWriter.java
@@ -30,7 +30,6 @@ import org.apache.solr.common.SolrException.ErrorCode;
import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.request.SolrQueryRequest;
-import org.apache.solr.response.transform.WriteableGeoJSON;
import org.apache.solr.schema.AbstractSpatialFieldType;
import org.apache.solr.schema.SchemaField;
import org.apache.solr.search.ReturnFields;
@@ -259,11 +258,7 @@ class GeoJSONWriter extends JSONWriter {
}
else if(geo instanceof IndexableField) {
str = ((IndexableField)geo).stringValue();
- }
- else if(geo instanceof WriteableGeoJSON) {
- shape = ((WriteableGeoJSON)geo).shape;
- }
- else {
+ } else {
str = geo.toString();
}
diff --git
a/solr/core/src/java/org/apache/solr/response/JSONResponseWriter.java
b/solr/core/src/java/org/apache/solr/response/JSONResponseWriter.java
index e83f923..3e5cc8b 100644
--- a/solr/core/src/java/org/apache/solr/response/JSONResponseWriter.java
+++ b/solr/core/src/java/org/apache/solr/response/JSONResponseWriter.java
@@ -204,6 +204,16 @@ class ArrayOfNameTypeValueJSONWriter extends JSONWriter {
}
@Override
+ public void writeStrRaw(String name, String val) throws IOException {
+ if (writeTypeAndValueKey) {
+ throw new IllegalStateException("NamedList should never be a field
value");
+ // and thus `writeTypeAndValueKey` should always have been cleared (set
to false) by the
+ // time `writeStrRaw(...)` is called (at the level of individual
SolrDocument fields).
+ }
+ super.writeStrRaw(name, val);
+ }
+
+ @Override
public void writeStr(String name, String val, boolean needsEscaping) throws
IOException {
ifNeededWriteTypeAndValueKey("str");
super.writeStr(name, val, needsEscaping);
@@ -230,9 +240,9 @@ class ArrayOfNameTypeValueJSONWriter extends JSONWriter {
}
@Override
- public void writeArray(String name, Iterator<?> val) throws IOException {
+ public void writeArray(String name, Iterator<?> val, boolean raw) throws
IOException {
ifNeededWriteTypeAndValueKey("array");
- super.writeArray(name, val);
+ super.writeArray(name, val, raw);
}
@Override
diff --git a/solr/core/src/java/org/apache/solr/response/JSONWriter.java
b/solr/core/src/java/org/apache/solr/response/JSONWriter.java
index 4e17696..d805cf2 100644
--- a/solr/core/src/java/org/apache/solr/response/JSONWriter.java
+++ b/solr/core/src/java/org/apache/solr/response/JSONWriter.java
@@ -103,7 +103,7 @@ public class JSONWriter extends TextResponseWriter
implements JsonTextWriter {
indent();
writeKey(fname, true);
Object val = doc.getFieldValue(fname);
- writeVal(fname, val);
+ writeVal(fname, val, shouldWriteRaw(fname, returnFields));
}
if(doc.hasChildDocuments()) {
diff --git a/solr/core/src/java/org/apache/solr/response/PHPResponseWriter.java
b/solr/core/src/java/org/apache/solr/response/PHPResponseWriter.java
index b10fb3d..ce2f781 100644
--- a/solr/core/src/java/org/apache/solr/response/PHPResponseWriter.java
+++ b/solr/core/src/java/org/apache/solr/response/PHPResponseWriter.java
@@ -78,8 +78,8 @@ class PHPWriter extends JSONWriter {
}
@Override
- public void writeArray(String name, List<?> l) throws IOException {
- writeArray(name,l.iterator());
+ public void writeArray(String name, List<?> l, boolean raw) throws
IOException {
+ writeArray(name,l.iterator(), raw);
}
@Override
diff --git
a/solr/core/src/java/org/apache/solr/response/PHPSerializedResponseWriter.java
b/solr/core/src/java/org/apache/solr/response/PHPSerializedResponseWriter.java
index 35b41af..b8c5882 100644
---
a/solr/core/src/java/org/apache/solr/response/PHPSerializedResponseWriter.java
+++
b/solr/core/src/java/org/apache/solr/response/PHPSerializedResponseWriter.java
@@ -168,7 +168,8 @@ class PHPSerializedWriter extends JSONWriter {
@Override
- public void writeArray(String name, Object[] val) throws IOException {
+ public void writeArray(String name, Object[] val, boolean raw) throws
IOException {
+ assert !raw;
writeMapOpener(val.length);
for(int i=0; i < val.length; i++) {
writeKey(i, false);
@@ -178,12 +179,13 @@ class PHPSerializedWriter extends JSONWriter {
}
@Override
- public void writeArray(String name, Iterator<?> val) throws IOException {
+ public void writeArray(String name, Iterator<?> val, boolean raw) throws
IOException {
+ assert !raw;
ArrayList<Object> vals = new ArrayList<>();
while( val.hasNext() ) {
vals.add(val.next());
}
- writeArray(name, vals.toArray());
+ writeArray(name, vals.toArray(), false);
}
@Override
diff --git
a/solr/core/src/java/org/apache/solr/response/RawShimTextResponseWriter.java
b/solr/core/src/java/org/apache/solr/response/RawShimTextResponseWriter.java
new file mode 100644
index 0000000..dd7b9cc
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/response/RawShimTextResponseWriter.java
@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.response;
+
+import org.apache.solr.common.SolrDocument;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.search.ReturnFields;
+
+import java.io.IOException;
+import java.util.Iterator;
+import java.util.Map;
+
+/**
+ * Utility class that delegates to another {@link TextResponseWriter}, but
converts normal write requests
+ * into "raw" requests that write field values directly to the delegate {@link
TextResponseWriter}'s backing writer.
+ */
+class RawShimTextResponseWriter extends TextResponseWriter {
+
+ private final TextResponseWriter backing;
+
+ RawShimTextResponseWriter(TextResponseWriter backing) {
+ super(null, false);
+ this.backing = backing;
+ }
+
+ // convert non-raw to raw. These are the reason this class exists (see class
javadocs)
+ @Override
+ public void writeStr(String name, String val, boolean needsEscaping) throws
IOException {
+ backing.writeStrRaw(name, val);
+ }
+
+ @Override
+ public void writeArray(String name, Iterator<?> val, boolean raw) throws
IOException {
+ backing.writeArray(name, val, true);
+ }
+
+ // Other stuff; just no-op delegation
+ @Override
+ public void writeStartDocumentList(String name, long start, int size, long
numFound, Float maxScore, Boolean numFoundExact) throws IOException {
+ backing.writeStartDocumentList(name, start, size, numFound, maxScore,
numFoundExact);
+ }
+
+ @Override
+ public void writeSolrDocument(String name, SolrDocument doc, ReturnFields
fields, int idx) throws IOException {
+ backing.writeSolrDocument(name, doc, fields, idx);
+ }
+
+ @Override
+ public void writeEndDocumentList() throws IOException {
+ backing.writeEndDocumentList();
+ }
+
+ @Override
+ public void writeMap(String name, Map<?, ?> val, boolean excludeOuter,
boolean isFirstVal) throws IOException {
+ backing.writeMap(name, val, excludeOuter, isFirstVal);
+ }
+
+ @Override
+ public void writeNull(String name) throws IOException {
+ backing.writeNull(name);
+ }
+
+ @Override
+ public void writeInt(String name, String val) throws IOException {
+ backing.writeInt(name, val);
+ }
+
+ @Override
+ public void writeLong(String name, String val) throws IOException {
+ backing.writeLong(name, val);
+ }
+
+ @Override
+ public void writeBool(String name, String val) throws IOException {
+ backing.writeBool(name, val);
+ }
+
+ @Override
+ public void writeFloat(String name, String val) throws IOException {
+ backing.writeFloat(name, val);
+ }
+
+ @Override
+ public void writeDouble(String name, String val) throws IOException {
+ backing.writeDouble(name, val);
+ }
+
+ @Override
+ public void writeDate(String name, String val) throws IOException {
+ backing.writeDate(name, val);
+ }
+
+ @Override
+ public void writeNamedList(String name, NamedList<?> val) throws IOException
{
+ backing.writeNamedList(name, val);
+ }
+}
diff --git a/solr/core/src/java/org/apache/solr/response/SchemaXmlWriter.java
b/solr/core/src/java/org/apache/solr/response/SchemaXmlWriter.java
index 7ab424d..bdc08c6 100644
--- a/solr/core/src/java/org/apache/solr/response/SchemaXmlWriter.java
+++ b/solr/core/src/java/org/apache/solr/response/SchemaXmlWriter.java
@@ -365,17 +365,17 @@ public class SchemaXmlWriter extends TextResponseWriter {
}
@Override
- public void writeArray(String name, Object[] val) throws IOException {
- writeArray(name, Arrays.asList(val).iterator());
+ public void writeArray(String name, Object[] val, boolean raw) throws
IOException {
+ writeArray(name, Arrays.asList(val).iterator(), raw);
}
@Override
- public void writeArray(String name, Iterator<?> iter) throws IOException {
+ public void writeArray(String name, Iterator<?> iter, boolean raw) throws
IOException {
if( iter.hasNext() ) {
startTag("arr", name, false );
incLevel();
while( iter.hasNext() ) {
- writeVal(null, iter.next());
+ writeVal(null, iter.next(), raw);
}
decLevel();
if (doIndent) indent();
diff --git
a/solr/core/src/java/org/apache/solr/response/TabularResponseWriter.java
b/solr/core/src/java/org/apache/solr/response/TabularResponseWriter.java
index 065d115..2148107 100644
--- a/solr/core/src/java/org/apache/solr/response/TabularResponseWriter.java
+++ b/solr/core/src/java/org/apache/solr/response/TabularResponseWriter.java
@@ -139,7 +139,7 @@ public abstract class TabularResponseWriter extends
TextResponseWriter {
}
@Override
- public void writeArray(String name, Iterator<?> val) throws IOException {
+ public void writeArray(String name, Iterator<?> val, boolean raw) throws
IOException {
}
@Override
diff --git
a/solr/core/src/java/org/apache/solr/response/TextResponseWriter.java
b/solr/core/src/java/org/apache/solr/response/TextResponseWriter.java
index e19906a..9ed8be9 100644
--- a/solr/core/src/java/org/apache/solr/response/TextResponseWriter.java
+++ b/solr/core/src/java/org/apache/solr/response/TextResponseWriter.java
@@ -19,7 +19,11 @@ package org.apache.solr.response;
import java.io.IOException;
import java.io.Writer;
import java.util.Calendar;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashSet;
import java.util.Iterator;
+import java.util.Set;
import org.apache.lucene.document.Document;
import org.apache.lucene.index.IndexableField;
@@ -31,10 +35,12 @@ import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.util.FastWriter;
import org.apache.solr.common.util.TextWriter;
import org.apache.solr.request.SolrQueryRequest;
+import org.apache.solr.response.transform.DocTransformer;
import org.apache.solr.schema.IndexSchema;
import org.apache.solr.schema.SchemaField;
import org.apache.solr.search.DocList;
import org.apache.solr.search.ReturnFields;
+import org.apache.solr.search.SolrReturnFields;
/** Base class for text-oriented response writers.
*
@@ -55,6 +61,15 @@ public abstract class TextResponseWriter implements
TextWriter {
protected Calendar cal; // reusable calendar instance
+ /**
+ * A signal object that must be used to differentiate from <code>null</code>
in strict object equality
+ * checks against {@link #rawReturnFields}, in order to determine the
appropriate context in which to
+ * write raw field values.
+ */
+ private static final ReturnFields NO_RAW_FIELDS = new SolrReturnFields();
+ private final TextResponseWriter rawShim;
+ private final Set<String> rawFields;
+ private final ReturnFields rawReturnFields;
public TextResponseWriter(Writer writer, SolrQueryRequest req,
SolrQueryResponse rsp) {
this.writer = writer == null ? null: FastWriter.wrap(writer);
@@ -67,6 +82,17 @@ public abstract class TextResponseWriter implements
TextWriter {
}
returnFields = rsp.getReturnFields();
if (req.getParams().getBool(CommonParams.OMIT_HEADER, false))
rsp.removeResponseHeader();
+ DocTransformer rootDocTransformer = returnFields.getTransformer();
+ Collection<String> rawFields;
+ if (rootDocTransformer == null || (rawFields =
rootDocTransformer.getRawFields()).isEmpty()) {
+ this.rawFields = null;
+ this.rawShim = null;
+ this.rawReturnFields = NO_RAW_FIELDS;
+ } else {
+ this.rawFields = rawFields.size() == 1 ?
Collections.singleton(rawFields.iterator().next()) : new HashSet<>(rawFields);
+ this.rawShim = new RawShimTextResponseWriter(this);
+ this.rawReturnFields = returnFields;
+ }
}
//only for test purposes
TextResponseWriter(Writer writer, boolean indent) {
@@ -76,6 +102,16 @@ public abstract class TextResponseWriter implements
TextWriter {
this.rsp = null;
returnFields = null;
this.doIndent = indent;
+ this.rawShim = null;
+ this.rawFields = null;
+ this.rawReturnFields = null;
+ }
+
+ /**
+ * NOTE: strict object equality check against {@link #rawReturnFields}; see
javadocs for {@link #NO_RAW_FIELDS}
+ */
+ protected final boolean shouldWriteRaw(String fname, ReturnFields
returnFields) {
+ return rawReturnFields == returnFields && rawFields.contains(fname);
}
/** done with this ResponseWriter... make sure any buffers are flushed to
writer */
@@ -106,7 +142,7 @@ public abstract class TextResponseWriter implements
TextWriter {
}
- public final void writeVal(String name, Object val) throws IOException {
+ public final void writeVal(String name, Object val, boolean raw) throws
IOException {
// if there get to be enough types, perhaps hashing on the type
// to get a handler might be faster (but types must be exact to do that...)
@@ -122,9 +158,10 @@ public abstract class TextResponseWriter implements
TextWriter {
IndexableField f = (IndexableField)val;
SchemaField sf = schema.getFieldOrNull( f.name() );
if( sf != null ) {
- sf.getType().write(this, name, f);
- }
- else {
+ sf.getType().write(raw ? rawShim : this, name, f);
+ } else if (raw) {
+ writeStrRaw(name, f.stringValue());
+ } else {
writeStr(name, f.stringValue(), true);
}
} else if (val instanceof Document) {
@@ -150,7 +187,7 @@ public abstract class TextResponseWriter implements
TextWriter {
BytesRef arr = (BytesRef)val;
writeByteArr(name, arr.bytes, arr.offset, arr.length);
} else {
- TextWriter.super.writeVal(name, val);
+ TextWriter.super.writeVal(name, val, raw);
}
}
// names are passed when writing primitives like writeInt to allow many
different
diff --git a/solr/core/src/java/org/apache/solr/response/XMLWriter.java
b/solr/core/src/java/org/apache/solr/response/XMLWriter.java
index e22cb8c..417938d 100644
--- a/solr/core/src/java/org/apache/solr/response/XMLWriter.java
+++ b/solr/core/src/java/org/apache/solr/response/XMLWriter.java
@@ -207,7 +207,7 @@ public class XMLWriter extends TextResponseWriter {
log.debug(String.valueOf(val));
}
}
- writeVal(fname, val);
+ writeVal(fname, val, shouldWriteRaw(fname, returnFields));
}
if(doc.hasChildDocuments()) {
@@ -299,17 +299,17 @@ public class XMLWriter extends TextResponseWriter {
}
@Override
- public void writeArray(String name, Object[] val) throws IOException {
- writeArray(name, Arrays.asList(val).iterator());
+ public void writeArray(String name, Object[] val, boolean raw) throws
IOException {
+ writeArray(name, Arrays.asList(val).iterator(), raw);
}
@Override
- public void writeArray(String name, Iterator<?> iter) throws IOException {
+ public void writeArray(String name, Iterator<?> iter, boolean raw) throws
IOException {
if( iter.hasNext() ) {
startTag("arr", name, false );
incLevel();
while( iter.hasNext() ) {
- writeVal(null, iter.next());
+ writeVal(null, iter.next(), raw);
}
decLevel();
if (doIndent) indent();
@@ -321,7 +321,7 @@ public class XMLWriter extends TextResponseWriter {
}
@Override
- public void writeIterator(String name, IteratorWriter val) throws
IOException {
+ public void writeIterator(String name, IteratorWriter val, boolean raw)
throws IOException {
// As the size is not known. So, always both startTag and endTag is written
// irrespective of number of entries in IteratorWriter
startTag("arr", name, false );
@@ -330,7 +330,7 @@ public class XMLWriter extends TextResponseWriter {
val.writeIter(new IteratorWriter.ItemWriter() {
@Override
public IteratorWriter.ItemWriter add(Object o) throws IOException {
- writeVal(null, o);
+ writeVal(null, o, raw);
return this;
}
});
@@ -352,6 +352,15 @@ public class XMLWriter extends TextResponseWriter {
}
@Override
+ public void writeStrRaw(String name, String val) throws IOException {
+ int contentLen = val == null ? 0 : val.length();
+ startTag("raw", name, contentLen == 0);
+ if (contentLen == 0) return;
+ writer.write(val, 0, contentLen);
+ writer.write("</raw>");
+ }
+
+ @Override
public void writeStr(String name, String val, boolean escape) throws
IOException {
writePrim("str",name,val,escape);
}
diff --git
a/solr/core/src/java/org/apache/solr/response/transform/DocTransformer.java
b/solr/core/src/java/org/apache/solr/response/transform/DocTransformer.java
index f1f6bf3..ce329f0 100644
--- a/solr/core/src/java/org/apache/solr/response/transform/DocTransformer.java
+++ b/solr/core/src/java/org/apache/solr/response/transform/DocTransformer.java
@@ -17,6 +17,8 @@
package org.apache.solr.response.transform;
import java.io.IOException;
+import java.util.Collection;
+import java.util.Collections;
import org.apache.solr.common.SolrDocument;
import org.apache.solr.response.QueryResponseWriter;
@@ -53,6 +55,22 @@ public abstract class DocTransformer {
}
/**
+ * If this transformer wants to bypass escaping in the {@link
org.apache.solr.response.TextResponseWriter} and
+ * write content directly to output for certain field(s), the names of any
such field(s) should be returned
+ *
+ * NOTE: normally this will be conditional on the `wt` param in the request,
as supplied to the
+ * {@link DocTransformer}'s parent {@link TransformerFactory} at the time of
transformer creation.
+ *
+ * @return Collection containing field names to be written raw; if no field
names should
+ * be written raw, an empty collection should be returned. Any collection
returned collection
+ * need not be externally modifiable -- i.e., {@link
java.util.Collections#singleton(Object)} is
+ * acceptable.
+ */
+ public Collection<String> getRawFields() {
+ return Collections.emptySet();
+ }
+
+ /**
* Indicates if this transformer requires access to the underlying index to
perform it's functions.
*
* In some situations (notably RealTimeGet) this method <i>may</i> be called
before {@link #setContext}
diff --git
a/solr/core/src/java/org/apache/solr/response/transform/DocTransformers.java
b/solr/core/src/java/org/apache/solr/response/transform/DocTransformers.java
index 68a0b9f..1bb7bc3 100644
--- a/solr/core/src/java/org/apache/solr/response/transform/DocTransformers.java
+++ b/solr/core/src/java/org/apache/solr/response/transform/DocTransformers.java
@@ -18,8 +18,10 @@ package org.apache.solr.response.transform;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Collection;
import java.util.Iterator;
import java.util.List;
+import java.util.stream.Collectors;
import org.apache.solr.common.SolrDocument;
import org.apache.solr.response.ResultContext;
@@ -49,6 +51,13 @@ public class DocTransformers extends DocTransformer
return str.toString();
}
+ @Override
+ public Collection<String> getRawFields() {
+ return children.stream().map(DocTransformer::getRawFields)
+ .flatMap(Collection::stream)
+ .collect(Collectors.toList());
+ }
+
public void addTransformer( DocTransformer a ) {
children.add( a );
}
diff --git
a/solr/core/src/java/org/apache/solr/response/transform/GeoTransformerFactory.java
b/solr/core/src/java/org/apache/solr/response/transform/GeoTransformerFactory.java
index 790d938..3c4dc53 100644
---
a/solr/core/src/java/org/apache/solr/response/transform/GeoTransformerFactory.java
+++
b/solr/core/src/java/org/apache/solr/response/transform/GeoTransformerFactory.java
@@ -17,7 +17,11 @@
package org.apache.solr.response.transform;
import java.io.IOException;
+import java.util.Collection;
+import java.util.Collections;
import java.util.Iterator;
+import java.util.Map;
+import java.util.Set;
import org.apache.lucene.index.IndexableField;
import org.apache.lucene.index.LeafReaderContext;
@@ -63,10 +67,15 @@ import org.locationtech.spatial4j.shape.Shape;
* </ul>
*
*/
-public class GeoTransformerFactory extends TransformerFactory
-{
+public class GeoTransformerFactory extends TransformerFactory implements
TransformerFactory.FieldRenamer {
+
@Override
public DocTransformer create(String display, SolrParams params,
SolrQueryRequest req) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public DocTransformer create(String display, SolrParams params,
SolrQueryRequest req, Map<String, String> renamedFields, Set<String>
reqFieldNames) {
String fname = params.get("f", display);
if(fname.startsWith("[") && fname.endsWith("]")) {
@@ -124,12 +133,7 @@ public class GeoTransformerFactory extends
TransformerFactory
// Using ValueSource
if(shapes!=null) {
- return new DocTransformer() {
- @Override
- public String getName() {
- return display;
- }
-
+ return new GeoDocTransformer(updater) {
@Override
public void transform(SolrDocument doc, int docid) throws IOException {
int leafOrd = ReaderUtil.subIndex(docid,
context.getSearcher().getTopReaderContext().leaves());
@@ -144,88 +148,104 @@ public class GeoTransformerFactory extends
TransformerFactory
}
+ // if source has been renamed, update reference
+ updater.field = renamedFields.getOrDefault(updater.field, updater.field);
+
+ // don't remove fields that were explicitly requested by others
+ final boolean copy = reqFieldNames != null &&
reqFieldNames.contains(updater.field);
+ if (!copy) {
+ renamedFields.put(updater.field, updater.display);
+ }
+
// Using the raw stored values
- return new DocTransformer() {
+ return new GeoDocTransformer(updater) {
@Override
public void transform(SolrDocument doc, int docid) throws IOException {
- Object val = doc.remove(updater.field);
+ Object val = copy ? doc.get(updater.field) : doc.remove(updater.field);
if(val!=null) {
updater.setValue(doc, val);
}
}
@Override
- public String getName() {
- return updater.display;
- }
-
- @Override
public String[] getExtraRequestFields() {
return new String[] {updater.field};
}
};
}
-}
+ private static abstract class GeoDocTransformer extends DocTransformer {
-class GeoFieldUpdater {
- String field;
- String display;
- String display_error;
-
- boolean isJSON;
- ShapeWriter writer;
- SupportedFormats formats;
-
- void addShape(SolrDocument doc, Shape shape) {
- if(isJSON) {
- doc.addField(display, new WriteableGeoJSON(shape, writer));
+ private final GeoFieldUpdater updater;
+
+ private GeoDocTransformer(GeoFieldUpdater updater) {
+ this.updater = updater;
}
- else {
- doc.addField(display, writer.toString(shape));
+
+ @Override
+ public String getName() {
+ return updater.display;
}
- }
-
- void setValue(SolrDocument doc, Object val) {
- doc.remove(display);
- if(val != null) {
- if(val instanceof Iterable) {
- Iterator<?> iter = ((Iterable<?>)val).iterator();
- while(iter.hasNext()) {
- addValue(doc, iter.next());
- }
- }
- else {
- addValue(doc, val);
- }
+
+ @Override
+ public Collection<String> getRawFields() {
+ return updater.isJSON ? Collections.singleton(updater.display) :
Collections.emptySet();
}
}
-
- void addValue(SolrDocument doc, Object val) {
- if(val == null) {
- return;
- }
-
- if(val instanceof Shape) {
- addShape(doc, (Shape)val);
+
+ private static class GeoFieldUpdater {
+ String field;
+ String display;
+ String display_error;
+
+ boolean isJSON;
+ ShapeWriter writer;
+ SupportedFormats formats;
+
+ void addShape(SolrDocument doc, Shape shape) {
+ doc.addField(display, writer.toString(shape));
}
- // Don't explode on 'InvalidShpae'
- else if( val instanceof Exception) {
- doc.setField( display_error, ((Exception)val).toString() );
+
+ void setValue(SolrDocument doc, Object val) {
+ doc.remove(display);
+ if(val != null) {
+ if(val instanceof Iterable) {
+ Iterator<?> iter = ((Iterable<?>)val).iterator();
+ while(iter.hasNext()) {
+ addValue(doc, iter.next());
+ }
+ }
+ else {
+ addValue(doc, val);
+ }
+ }
}
- else {
- // Use the stored value
- if(val instanceof IndexableField) {
- val = ((IndexableField)val).stringValue();
+
+ void addValue(SolrDocument doc, Object val) {
+ if(val == null) {
+ return;
+ }
+
+ if(val instanceof Shape) {
+ addShape(doc, (Shape)val);
}
- try {
- addShape(doc, formats.read(val.toString()));
+ // Don't explode on 'InvalidShpae'
+ else if( val instanceof Exception) {
+ doc.setField( display_error, ((Exception)val).toString() );
}
- catch(Exception ex) {
- doc.setField( display_error, ex.toString() );
+ else {
+ // Use the stored value
+ if(val instanceof IndexableField) {
+ val = ((IndexableField)val).stringValue();
+ }
+ try {
+ addShape(doc, formats.read(val.toString()));
+ }
+ catch(Exception ex) {
+ doc.setField( display_error, ex.toString() );
+ }
}
}
}
}
-
diff --git
a/solr/core/src/java/org/apache/solr/response/transform/RawValueTransformerFactory.java
b/solr/core/src/java/org/apache/solr/response/transform/RawValueTransformerFactory.java
index 5838e10..da657bb 100644
---
a/solr/core/src/java/org/apache/solr/response/transform/RawValueTransformerFactory.java
+++
b/solr/core/src/java/org/apache/solr/response/transform/RawValueTransformerFactory.java
@@ -16,27 +16,23 @@
*/
package org.apache.solr.response.transform;
-import java.io.IOException;
-import java.util.ArrayList;
import java.util.Collection;
+import java.util.Collections;
+import java.util.Map;
+import java.util.Set;
import com.google.common.base.Strings;
-import org.apache.lucene.index.IndexableField;
import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.params.SolrParams;
-import org.apache.solr.common.util.JavaBinCodec;
-import org.apache.solr.common.util.JavaBinCodec.ObjectResolver;
import org.apache.solr.common.util.NamedList;
-import org.apache.solr.common.util.TextWriter;
-import org.apache.solr.common.util.WriteableValue;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.response.QueryResponseWriter;
/**
* @since solr 5.2
*/
-public class RawValueTransformerFactory extends TransformerFactory
+public class RawValueTransformerFactory extends TransformerFactory implements
TransformerFactory.FieldRenamer
{
String applyToWT = null;
@@ -58,10 +54,28 @@ public class RawValueTransformerFactory extends
TransformerFactory
@Override
public DocTransformer create(String display, SolrParams params,
SolrQueryRequest req) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public boolean mayModifyValue() {
+ // The only thing we may modify is the _serialization_; field values per
se are guaranteed to be unmodified.
+ return false;
+ }
+
+ @Override
+ public DocTransformer create(String display, SolrParams params,
SolrQueryRequest req,
+ Map<String, String> renamedFields, Set<String>
reqFieldNames) {
String field = params.get("f");
if(Strings.isNullOrEmpty(field)) {
field = display;
}
+ field = renamedFields.getOrDefault(field, field);
+ final boolean rename = !field.equals(display);
+ final boolean copy = rename && reqFieldNames != null &&
reqFieldNames.contains(field);
+ if (!copy) {
+ renamedFields.put(field, display);
+ }
// When a 'wt' is specified in the transformer, only apply it to the same
wt
boolean apply = true;
if(applyToWT!=null) {
@@ -79,25 +93,27 @@ public class RawValueTransformerFactory extends
TransformerFactory
}
if(apply) {
- return new RawTransformer( field, display );
+ return new RawTransformer( field, display, copy );
}
- if (field.equals(display)) {
+ if (!rename) {
// we have to ensure the field is returned
return new DocTransformer.NoopFieldTransformer(field);
}
- return new RenameFieldTransformer( field, display, false );
+ return new RenameFieldTransformer( field, display, copy );
}
-
+
static class RawTransformer extends DocTransformer
{
final String field;
final String display;
+ final boolean copy;
- public RawTransformer( String field, String display )
+ public RawTransformer( String field, String display, boolean copy )
{
this.field = field;
this.display = display;
+ this.copy = copy;
}
@Override
@@ -107,57 +123,21 @@ public class RawValueTransformerFactory extends
TransformerFactory
}
@Override
- public void transform(SolrDocument doc, int docid) {
- Object val = doc.remove(field);
- if(val==null) {
- return;
- }
- if(val instanceof Collection) {
- Collection<?> current = (Collection<?>)val;
- ArrayList<WriteableStringValue> vals = new
ArrayList<RawValueTransformerFactory.WriteableStringValue>();
- for(Object v : current) {
- vals.add(new WriteableStringValue(v));
- }
- doc.setField(display, vals);
- }
- else {
- doc.setField(display, new WriteableStringValue(val));
- }
+ public Collection<String> getRawFields() {
+ return Collections.singleton(display);
}
@Override
- public String[] getExtraRequestFields() {
- return new String[] {this.field};
- }
- }
-
- public static class WriteableStringValue extends WriteableValue {
- public final Object val;
-
- public WriteableStringValue(Object val) {
- this.val = val;
- }
-
- @Override
- public void write(String name, TextWriter writer) throws IOException {
- String str = null;
- if(val instanceof IndexableField) { // delays holding it in memory
- str = ((IndexableField)val).stringValue();
- }
- else {
- str = val.toString();
+ public void transform(SolrDocument doc, int docid) {
+ Object val = copy ? doc.get(field) : doc.remove(field);
+ if(val != null) {
+ doc.setField(display, val);
}
- writer.getWriter().write(str);
}
@Override
- public Object resolve(Object o, JavaBinCodec codec) throws IOException {
- ObjectResolver orig = codec.getResolver();
- if(orig != null) {
- codec.writeVal(orig.resolve(val, codec));
- return null;
- }
- return val.toString();
+ public String[] getExtraRequestFields() {
+ return new String[] {this.field};
}
}
}
diff --git
a/solr/core/src/java/org/apache/solr/response/transform/RenameFieldTransformer.java
b/solr/core/src/java/org/apache/solr/response/transform/RenameFieldTransformer.java
index 41ac54f..d67ea41 100644
---
a/solr/core/src/java/org/apache/solr/response/transform/RenameFieldTransformer.java
+++
b/solr/core/src/java/org/apache/solr/response/transform/RenameFieldTransformer.java
@@ -29,12 +29,14 @@ public class RenameFieldTransformer extends DocTransformer
final String from;
final String to;
final boolean copy;
+ final String[] ensureFromFieldPresent;
- public RenameFieldTransformer( String from, String to, boolean copy )
- {
+ public RenameFieldTransformer( String from, String to, boolean copy ) {
this.from = from;
this.to = to;
this.copy = copy;
+ this.ensureFromFieldPresent = new String[] { from };
+ assert !from.equals(to);
}
@Override
@@ -50,4 +52,9 @@ public class RenameFieldTransformer extends DocTransformer
doc.setField(to, v);
}
}
+
+ @Override
+ public String[] getExtraRequestFields() {
+ return ensureFromFieldPresent;
+ }
}
diff --git
a/solr/core/src/java/org/apache/solr/response/transform/TransformerFactory.java
b/solr/core/src/java/org/apache/solr/response/transform/TransformerFactory.java
index 97134ea..ab9807f 100644
---
a/solr/core/src/java/org/apache/solr/response/transform/TransformerFactory.java
+++
b/solr/core/src/java/org/apache/solr/response/transform/TransformerFactory.java
@@ -18,6 +18,7 @@ package org.apache.solr.response.transform;
import java.util.HashMap;
import java.util.Map;
+import java.util.Set;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.NamedList;
@@ -40,6 +41,62 @@ public abstract class TransformerFactory implements
NamedListInitializedPlugin
public abstract DocTransformer create(String field, SolrParams params,
SolrQueryRequest req);
+ /**
+ * The {@link FieldRenamer} interface should be implemented by any {@link
TransformerFactory} capable of generating
+ * transformers that might rename fields, and should implement {@link
#create(String, SolrParams, SolrQueryRequest, Map, Set)}
+ * in place of {@link #create(String, SolrParams, SolrQueryRequest)} (with
the latter method
+ * overridden to throw {@link UnsupportedOperationException}).
+ *
+ * {@link DocTransformer}s returned via {@link #create(String, SolrParams,
SolrQueryRequest, Map, Set)}
+ * will be added in a second pass, allowing simplified logic in {@link
TransformerFactory#create(String, SolrParams, SolrQueryRequest)}
+ * for non-renaming factories.
+ *
+ * {@link #create(String, SolrParams, SolrQueryRequest, Map, Set)} must
implement extra logic to be aware of
+ * preceding field renames, and to make subsequent {@link FieldRenamer}
transformers aware of its own field renames.
+ *
+ * It is harmless for a {@link DocTransformer} that does _not_ in practice
rename fields to be returned from a
+ * factory that implements this interface (e.g., for conditional renames?);
but doing so opens the possibility of
+ * {@link #create(String, SolrParams, SolrQueryRequest, Map, Set)} being
called _after_ fields have been renamed,
+ * so such implementations must still check whether the field with which
they are concerned has been renamed ...
+ * and if it _has_, must copy the field back to its original name. This
situation also demonstrates the
+ * motivation for separating the creation of {@link DocTransformer}s into
two phases: an initial phase involving
+ * no field renames, and a subsequent phase that implement extra logic to
properly handle field renames.
+ */
+ public interface FieldRenamer {
+ // TODO: Behavior is undefined in the event of a "destination field"
collision (e.g., a user maps two fields to
+ // the same "destination field", or maps a field to a top-level requested
field). In the future, the easiest way
+ // to detect such a case would be by "failing fast" upon renaming to a
field that already has an associated value,
+ // or support for this feature could be expressly added via a hypothetical
+ // `combined_field:[consolidate fl=field_1,field_2]` transformer.
+ /**
+ * Analogous to {@link TransformerFactory#create(String, SolrParams,
SolrQueryRequest)}, but to be implemented
+ * by {@link TransformerFactory}s that produce {@link DocTransformer}s
that may rename fields.
+ *
+ * @param field The destination field
+ * @param params Local params associated with this transformer (e.g.,
source field)
+ * @param req The current request
+ * @param renamedFields Maps source=>dest renamed fields.
Implementations should check this first, updating
+ * their own "source" field(s) as necessary, and if
renaming (not copying) fields, should
+ * also update this map with the implementations
"own" introduced source=>dest field
+ * mapping
+ * @param reqFieldNames Set of explicitly requested field names;
implementations should consult this set to
+ * determine whether it's appropriate to rename (vs.
copy) a field (e.g.: <code>boolean
+ * copy = reqFieldNames != null &&
reqFieldNames.contains(sourceField)</code>)
+ * @return A transformer to be used in processing field values in returned
documents.
+ */
+ DocTransformer create(String field, SolrParams params, SolrQueryRequest
req, Map<String, String> renamedFields, Set<String> reqFieldNames);
+
+ /**
+ * Returns <code>true</code> if implementations of this class may (even
subtly) modify field values.
+ * ({@link GeoTransformerFactory} may do this, e.g.). To fail safe, the
default implementation returns
+ * <code>true</code>. This method should be overridden to return
<code>false</code> if the implementing
+ * class is guaranteed to not modify any values for the fields that it
renames.
+ */
+ default boolean mayModifyValue() {
+ return true;
+ }
+ }
+
public static final Map<String,TransformerFactory> defaultFactories = new
HashMap<>(9, 1.0f);
static {
defaultFactories.put( "explain", new ExplainAugmenterFactory() );
diff --git
a/solr/core/src/java/org/apache/solr/response/transform/WriteableGeoJSON.java
b/solr/core/src/java/org/apache/solr/response/transform/WriteableGeoJSON.java
deleted file mode 100644
index f89f5c8..0000000
---
a/solr/core/src/java/org/apache/solr/response/transform/WriteableGeoJSON.java
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.response.transform;
-
-import java.io.IOException;
-
-import org.apache.solr.common.util.JavaBinCodec;
-import org.apache.solr.common.util.TextWriter;
-import org.apache.solr.common.util.WriteableValue;
-import org.locationtech.spatial4j.io.ShapeWriter;
-import org.locationtech.spatial4j.shape.Shape;
-
-/**
- * This will let the writer add values to the response directly
- */
-public class WriteableGeoJSON extends WriteableValue {
-
- public final Shape shape;
- public final ShapeWriter jsonWriter;
-
- public WriteableGeoJSON(Shape shape, ShapeWriter jsonWriter) {
- this.shape = shape;
- this.jsonWriter = jsonWriter;
- }
-
- @Override
- public Object resolve(Object o, JavaBinCodec codec) throws IOException {
- codec.writeStr(jsonWriter.toString(shape));
- return null; // this means we wrote it
- }
-
- @Override
- public void write(String name, TextWriter writer) throws IOException {
- jsonWriter.write(writer.getWriter(), shape);
- }
-
- @Override
- public String toString() {
- return jsonWriter.toString(shape);
- }
-}
diff --git a/solr/core/src/java/org/apache/solr/search/SolrReturnFields.java
b/solr/core/src/java/org/apache/solr/search/SolrReturnFields.java
index 6135bb2..4903e0b 100644
--- a/solr/core/src/java/org/apache/solr/search/SolrReturnFields.java
+++ b/solr/core/src/java/org/apache/solr/search/SolrReturnFields.java
@@ -16,9 +16,12 @@
*/
package org.apache.solr.search;
+import java.util.ArrayDeque;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
+import java.util.Deque;
+import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashSet;
import java.util.List;
@@ -35,7 +38,6 @@ import org.apache.solr.common.SolrException;
import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.params.SolrParams;
-import org.apache.solr.common.util.NamedList;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.response.transform.DocTransformer;
import org.apache.solr.response.transform.DocTransformers;
@@ -160,6 +162,10 @@ public class SolrReturnFields extends ReturnFields {
}
+ /**
+ * Parsing is done in two passes (see javadocs for {@link
org.apache.solr.response.transform.TransformerFactory.FieldRenamer}
+ * for an explanation of the logic behind deferring creation of "rename
field" transformers).
+ */
private void parseFieldList(String[] fl, SolrQueryRequest req) {
_wantsScore = false;
_wantsAllFields = false;
@@ -168,32 +174,26 @@ public class SolrReturnFields extends ReturnFields {
return;
}
- NamedList<String> rename = new NamedList<>();
+ Deque<DeferredRenameEntry> deferredRenameAugmenters = new ArrayDeque<>();
DocTransformers augmenters = new DocTransformers();
for (String fieldList : fl) {
- add(fieldList,rename,augmenters,req);
+ add(fieldList,deferredRenameAugmenters,augmenters,req);
}
- for( int i=0; i<rename.size(); i++ ) {
- String from = rename.getName(i);
- String to = rename.getVal(i);
- okFieldNames.add( to );
- boolean copy = (reqFieldNames!=null && reqFieldNames.contains(from));
- if(!copy) {
- // Check that subsequent copy/rename requests have the field they need
to copy
- for(int j=i+1; j<rename.size(); j++) {
- if(from.equals(rename.getName(j))) {
- rename.setName(j, to); // copy from the current target
- if(reqFieldNames==null) {
- reqFieldNames = new LinkedHashSet<>();
- }
- reqFieldNames.add(to); // don't rename our current target
+ Map<String, String> renamedNotCopied = new HashMap<>();
+ for (DeferredRenameEntry e : deferredRenameAugmenters) {
+ DocTransformer t = e.create(renamedNotCopied, reqFieldNames);
+ augmenters.addTransformer(t);
+ if (!_wantsAllFields) {
+ final String[] extraRequestFields = t.getExtraRequestFields();
+ if (extraRequestFields != null) {
+ for (String f : extraRequestFields) {
+ fields.add(f);
}
}
}
- augmenters.addTransformer( new RenameFieldTransformer( from, to, copy )
);
}
- if (rename.size() > 0 ) {
- renameFields = rename.asShallowMap();
+ if (!renamedNotCopied.isEmpty()) {
+ renameFields = renamedNotCopied;
}
if( !_wantsAllFields && !globs.isEmpty() ) {
// TODO??? need to fill up the fields with matching field names in the
index
@@ -236,7 +236,7 @@ public class SolrReturnFields extends ReturnFields {
return null;
}
- private void add(String fl, NamedList<String> rename, DocTransformers
augmenters, SolrQueryRequest req) {
+ private void add(String fl, Deque<DeferredRenameEntry> deferred,
DocTransformers augmenters, SolrQueryRequest req) {
if( fl == null ) {
return;
}
@@ -278,8 +278,9 @@ public class SolrReturnFields extends ReturnFields {
field = sp.getId(null);
ch = sp.ch();
if (field != null && (Character.isWhitespace(ch) || ch == ',' ||
ch==0)) {
- rename.add(field, key);
- addField(field, key, augmenters, false);
+ deferred.addFirst(new DeferredRenameEntry(key, new
ModifiableSolrParams().set(SOURCE_FIELD_ARGNAME, field), req,
RENAME_FIELD_TRANSFORMER_FACTORY));
+ // NOTE: treat as pseudoField below because `fields` will be
modified on deferred invocation
+ addField(field, key, augmenters, true);
continue;
}
// an invalid field name... reset the position pointer to retry
@@ -326,7 +327,20 @@ public class SolrReturnFields extends ReturnFields {
}
TransformerFactory factory = req.getCore().getTransformerFactory(
augmenterName );
- if( factory != null ) {
+ if (factory instanceof TransformerFactory.FieldRenamer) {
+ // NOTE: `deferred` is a Deque because some TransformerFactories
(e.g., `GeoTransformerFactory`) can
+ // subtly modify the representation of the associated value (i.e.,
it's not just a straight rename). This
+ // subverts the "update source field" phase of
`FieldRenamer.create(...)`. We _know_ however that "simple"
+ // field renames don't do any value modification whatsoever, so
those are added to the beginning of
+ // the `deferred` Deque so that they will be processed first, and
all other `FieldRenamers` are
+ // added (here) to the front or back of the Deque, depending on
the return value of `mayModifyValue()`.
+ final DeferredRenameEntry deferredEntry = new
DeferredRenameEntry(disp, augmenterParams, req,
(TransformerFactory.FieldRenamer) factory);
+ if (((TransformerFactory.FieldRenamer) factory).mayModifyValue()) {
+ deferred.addLast(deferredEntry);
+ } else {
+ deferred.addFirst(deferredEntry);
+ }
+ } else if (factory != null) {
DocTransformer t = factory.create(disp, augmenterParams, req);
if(t!=null) {
if(!_wantsAllFields) {
@@ -415,7 +429,7 @@ public class SolrReturnFields extends ReturnFields {
// OK, it was an oddly named field
addField(field, key, augmenters, false);
if( key != null ) {
- rename.add(field, key);
+ deferred.addFirst(new DeferredRenameEntry(key, new
ModifiableSolrParams().set(SOURCE_FIELD_ARGNAME, field), req,
RENAME_FIELD_TRANSFORMER_FACTORY));
}
} else {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
"Error parsing fieldname: " + e.getMessage(), e);
@@ -430,6 +444,43 @@ public class SolrReturnFields extends ReturnFields {
}
}
+ private static final String SOURCE_FIELD_ARGNAME = "sourceField";
+ private static final TransformerFactory.FieldRenamer
RENAME_FIELD_TRANSFORMER_FACTORY = new TransformerFactory.FieldRenamer() {
+ @Override
+ public DocTransformer create(String to, SolrParams params,
SolrQueryRequest req, Map<String, String> renamedFields, Set<String>
reqFieldNames) {
+ String from = params.get(SOURCE_FIELD_ARGNAME);
+ from = renamedFields.getOrDefault(from, from);
+ final boolean copy = reqFieldNames != null &&
reqFieldNames.contains(from);
+ if (!copy) {
+ renamedFields.put(from, to);
+ }
+ return new RenameFieldTransformer(from, to, copy);
+ }
+
+ @Override
+ public boolean mayModifyValue() {
+ return false;
+ }
+ };
+
+ private static final class DeferredRenameEntry {
+ private final String field;
+ private final SolrParams params;
+ private final SolrQueryRequest req;
+ private final TransformerFactory.FieldRenamer factory;
+
+ private DeferredRenameEntry(String field, SolrParams params,
SolrQueryRequest req, TransformerFactory.FieldRenamer factory) {
+ this.field = field;
+ this.params = params;
+ this.req = req;
+ this.factory = factory;
+ }
+
+ private DocTransformer create(Map<String, String> renamedFields,
Set<String> reqFieldNames) {
+ return factory.create(field, params, req, renamedFields, reqFieldNames);
+ }
+ }
+
private void addField(String field, String key, DocTransformers augmenters,
boolean isPseudoField)
{
if(reqFieldNames==null) {
diff --git a/solr/core/src/test/org/apache/solr/cloud/TestRandomFlRTGCloud.java
b/solr/core/src/test/org/apache/solr/cloud/TestRandomFlRTGCloud.java
index 34b0279..26b2fdf 100644
--- a/solr/core/src/test/org/apache/solr/cloud/TestRandomFlRTGCloud.java
+++ b/solr/core/src/test/org/apache/solr/cloud/TestRandomFlRTGCloud.java
@@ -17,6 +17,7 @@
package org.apache.solr.cloud;
import java.io.IOException;
+import java.io.StringReader;
import java.lang.invoke.MethodHandles;
import java.nio.file.Path;
import java.util.ArrayList;
@@ -33,16 +34,21 @@ import java.util.TreeSet;
import org.apache.commons.io.FilenameUtils;
import org.apache.lucene.util.TestUtil;
+import org.apache.solr.client.solrj.ResponseParser;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.embedded.JettySolrRunner;
import org.apache.solr.client.solrj.impl.CloudSolrClient;
import org.apache.solr.client.solrj.impl.HttpSolrClient;
+import org.apache.solr.client.solrj.impl.NoOpResponseParser;
+import org.apache.solr.client.solrj.impl.XMLResponseParser;
import org.apache.solr.client.solrj.request.CollectionAdminRequest;
+import org.apache.solr.client.solrj.request.QueryRequest;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.SolrDocumentList;
import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.NamedList;
@@ -52,9 +58,14 @@ import org.apache.solr.response.transform.TransformerFactory;
import org.apache.solr.util.RandomizeSSL;
import org.junit.AfterClass;
import org.junit.BeforeClass;
+import org.noggit.ObjectBuilder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import javax.xml.stream.XMLStreamConstants;
+import javax.xml.stream.XMLStreamException;
+import javax.xml.stream.XMLStreamReader;
+
/** @see TestCloudPseudoReturnFields */
@RandomizeSSL(clientAuth=0.0,reason="client auth uses too much RAM")
public class TestRandomFlRTGCloud extends SolrCloudTestCase {
@@ -91,18 +102,28 @@ public class TestRandomFlRTGCloud extends
SolrCloudTestCase {
new ValueAugmenterValidator(1976, "val_alias"),
//
new RenameFieldValueValidator("id", "my_id_alias"),
+ // NOTE: we add a SimpleFieldValueValidator below to check that we can
enforce the presence of this field,
+ // even when it may have been "renamed" by the transformer above? (this
and other such instances are
+ // marked with `//REQ`); also add a RenameFieldValueValidator to "fork"
values, marked with `//FORK`.
+ new SimpleFieldValueValidator("id"), //REQ
new SimpleFieldValueValidator("aaa_i"),
new RenameFieldValueValidator("bbb_i", "my_int_field_alias"),
+ new RenameFieldValueValidator("bbb_i", "my_int_field_alias2"), //FORK
+ new SimpleFieldValueValidator("bbb_i"), //REQ
new SimpleFieldValueValidator("ccc_s"),
new RenameFieldValueValidator("ddd_s", "my_str_field_alias"),
- //
- // SOLR-9376: RawValueTransformerFactory doesn't work in cloud mode
- //
- // new RawFieldValueValidator("json", "eee_s", "my_json_field_alias"),
- // new RawFieldValueValidator("json", "fff_s"),
- // new RawFieldValueValidator("xml", "ggg_s", "my_xml_field_alias"),
- // new RawFieldValueValidator("xml", "hhh_s"),
- //
+ new RenameFieldValueValidator("ddd_s", "my_str_field_alias2"), // FORK
+ new SimpleFieldValueValidator("ddd_s"), //REQ
+
+ new RawFieldValueValidator("json", "eee_s", "my_json_field_alias"),
+ new RenameFieldValueValidator("eee_s", "my_escaped_json_field_alias"),
// FORK
+ new SimpleFieldValueValidator("eee_s"), //REQ
+ new RawFieldValueValidator("json", "fff_s"),
+ new RawFieldValueValidator("xml", "ggg_s", "my_xml_field_alias"),
+ new RenameFieldValueValidator("ggg_s", "my_escaped_xml_field_alias"), //
FORK
+ new SimpleFieldValueValidator("ggg_s"), //REQ
+ new RawFieldValueValidator("xml", "hhh_s"),
+
new NotIncludedValidator("bogus_unused_field_ss"),
new
NotIncludedValidator("bogus_alias","bogus_alias:other_bogus_field_i"),
new NotIncludedValidator("bogus_raw_alias","bogus_raw_alias:[xml
f=bogus_raw_field_ss]"),
@@ -111,6 +132,8 @@ public class TestRandomFlRTGCloud extends SolrCloudTestCase
{
new FunctionValidator("aaa_i", "func_aaa_alias"),
new GeoTransformerValidator("geo_1_srpt"),
new GeoTransformerValidator("geo_2_srpt","my_geo_alias"),
+ new RenameFieldValueValidator("geo_2_srpt", "my_geo_alias2"), // FORK
+ new SimpleFieldValueValidator("geo_2_srpt"), //REQ
new ExplainValidator(),
new ExplainValidator("explain_alias"),
new SubQueryValidator(),
@@ -144,7 +167,7 @@ public class TestRandomFlRTGCloud extends SolrCloudTestCase
{
.withProperty("schema", "schema-pseudo-fields.xml")
.process(CLOUD_CLIENT);
- cluster.waitForActiveCollection(COLLECTION_NAME, numShards, repFactor *
numShards);
+ cluster.waitForActiveCollection(COLLECTION_NAME, numShards, repFactor *
numShards);
for (JettySolrRunner jetty : cluster.getJettySolrRunners()) {
CLIENTS.add(getHttpSolrClient(jetty.getBaseUrl() + "/" + COLLECTION_NAME
+ "/"));
@@ -187,7 +210,7 @@ public class TestRandomFlRTGCloud extends SolrCloudTestCase
{
// items should only be added to this list if it's known that they do not
work with RTG
// and a specific Jira for fixing this is listed as a comment
final List<String> knownBugs = Arrays.asList
- ( "xml","json", // SOLR-9376
+ (
"child" // way to complicatd to vet with this test, see SOLR-9379
instead
);
@@ -318,10 +341,10 @@ public class TestRandomFlRTGCloud extends
SolrCloudTestCase {
//
"ccc_s",
TestUtil.randomSimpleString(random()),
"ddd_s",
TestUtil.randomSimpleString(random()),
- "eee_s",
TestUtil.randomSimpleString(random()),
- "fff_s",
TestUtil.randomSimpleString(random()),
- "ggg_s",
TestUtil.randomSimpleString(random()),
- "hhh_s",
TestUtil.randomSimpleString(random()),
+ "eee_s",
makeJson(TestUtil.randomSimpleString(random())),
+ "fff_s",
makeJson(TestUtil.randomSimpleString(random())),
+ "ggg_s",
makeXml(TestUtil.randomSimpleString(random())),
+ "hhh_s",
makeXml(TestUtil.randomSimpleString(random())),
//
"geo_1_srpt",
GeoTransformerValidator.getValueForIndexing(random()),
"geo_2_srpt",
GeoTransformerValidator.getValueForIndexing(random()),
@@ -338,7 +361,60 @@ public class TestRandomFlRTGCloud extends
SolrCloudTestCase {
return doc;
}
-
+ private String makeJson(String s) {
+ switch (random().nextInt(3)) {
+ case 0:
+ // simple string
+ return '"' + s + '"';
+ case 1:
+ // array
+ return "[\"" + s + "\", \"" + s + "\"]";
+ case 2:
+ // map
+ return "{\"" + s + "\":\"" + s + "\"}";
+ default:
+ throw new IllegalStateException();
+ }
+ }
+
+ private String makeXml(String s) {
+ switch (random().nextInt(3)) {
+ case 0:
+ // simple string
+ return s;
+ case 1:
+ // simple element
+ return "<root>" + s + "</root>";
+ case 2:
+ // slightly more complex
+ return "<root><inner1>" + s + "</inner1><inner2>" + s +
"</inner2></root>";
+ default:
+ throw new IllegalStateException();
+ }
+ }
+
+ private static final ResponseParser RAW_XML_RESPONSE_PARSER = new
NoOpResponseParser();
+ private static final ResponseParser RAW_JSON_RESPONSE_PARSER = new
NoOpResponseParser() {
+ @Override
+ public String getWriterType() {
+ return "json";
+ }
+ };
+
+ private static ResponseParser modifyParser(HttpSolrClient client, final
String wt) {
+ final ResponseParser ret = client.getParser();
+ switch (wt) {
+ case "xml":
+ client.setParser(RAW_XML_RESPONSE_PARSER);
+ return ret;
+ case "json":
+ client.setParser(RAW_JSON_RESPONSE_PARSER);
+ return ret;
+ default:
+ return null;
+ }
+ }
+
/**
* Does one or more RTG request for the specified docIds with a randomized
fl & fq params, asserting
* that the returned document (if any) makes sense given the expected
SolrInputDocuments
@@ -397,11 +473,44 @@ public class TestRandomFlRTGCloud extends
SolrCloudTestCase {
assert 1 == idsToRequest.size();
params.add("id",idsToRequest.get(0));
}
-
- final QueryResponse rsp = client.query(params);
- assertNotNull(params.toString(), rsp);
- final SolrDocumentList docs = getDocsFromRTGResponse(askForList, rsp);
+ String wt = params.get(CommonParams.WT, "javabin");
+ final ResponseParser restoreResponseParser;
+ if (client instanceof HttpSolrClient) {
+ restoreResponseParser = modifyParser((HttpSolrClient) client, wt);
+ } else {
+ // unless HttpSolrClient, `wt` doesn't matter -- it'll always be binary.
+ wt = "javabin";
+ restoreResponseParser = null;
+ }
+
+ final Object rsp;
+ final SolrDocumentList docs;
+ if ("javabin".equals(wt)) {
+ // the most common case
+ final QueryResponse qRsp = client.query(params);
+ assertNotNull(params.toString(), qRsp);
+ rsp = qRsp;
+ docs = getDocsFromRTGResponse(askForList, qRsp);
+ } else {
+ final NamedList<Object> nlRsp = client.request(new QueryRequest(params));
+ assertNotNull(restoreResponseParser);
+ ((HttpSolrClient) client).setParser(restoreResponseParser);
+ assertNotNull(params.toString(), nlRsp);
+ rsp = nlRsp;
+ final String textResult = (String) nlRsp.get("response");
+ switch (wt) {
+ case "json":
+ docs = getDocsFromJsonResponse(askForList, textResult);
+ break;
+ case "xml":
+ docs = getDocsFromXmlResponse(askForList, textResult);
+ break;
+ default:
+ throw new IllegalStateException();
+ }
+ }
+
assertNotNull(params + " => " + rsp, docs);
assertEquals("num docs mismatch: " + params + " => " + docsToExpect + " vs
" + docs,
@@ -416,7 +525,7 @@ public class TestRandomFlRTGCloud extends SolrCloudTestCase
{
Set<String> expectedFieldNames = new TreeSet<>();
for (FlValidator v : validators) {
- expectedFieldNames.addAll(v.assertRTGResults(validators, expected,
actual));
+ expectedFieldNames.addAll(v.assertRTGResults(validators, expected,
actual, wt));
}
// ensure only expected field names are in the actual document
Set<String> actualFieldNames = new TreeSet<>(actual.getFieldNames());
@@ -450,8 +559,36 @@ public class TestRandomFlRTGCloud extends
SolrCloudTestCase {
}
return result;
}
-
- /**
+
+ @SuppressWarnings("unchecked")
+ private static SolrDocumentList getSolrDocumentList(Map<String, Object>
response) {
+ SolrDocumentList ret = new SolrDocumentList();
+ for (Map<String, Object> doc : (List<Map<String, Object>>)
response.get("docs")) {
+ ret.add(new SolrDocument(doc));
+ }
+ return ret;
+ }
+
+ @SuppressWarnings("unchecked")
+ private static SolrDocumentList getDocsFromJsonResponse(final boolean
expectList, final String rsp) throws IOException {
+ Map<String, Object> nl = (Map<String, Object>) ObjectBuilder.fromJSON(rsp);
+ if (expectList) {
+ return getSolrDocumentList((Map<String, Object>) nl.get("response"));
+ } else {
+ SolrDocumentList ret = new SolrDocumentList();
+ Map<String, Object> doc = (Map<String, Object>) nl.get("doc");
+ if (doc != null) {
+ ret.add(new SolrDocument(doc));
+ }
+ return ret;
+ }
+ }
+
+ private static SolrDocumentList getDocsFromXmlResponse(final boolean
expectList, final String rsp) {
+ return getDocsFromRTGResponse(expectList, new QueryResponse(new
RawCapableXMLResponseParser().processResponse(new StringReader(rsp)), null));
+ }
+
+ /**
* returns a random SolrClient -- either a CloudSolrClient, or an
HttpSolrClient pointed
* at a node in our cluster
*/
@@ -531,11 +668,13 @@ public class TestRandomFlRTGCloud extends
SolrCloudTestCase {
* @param validators all validators in use for this request, including the
current one
* @param expected a document containing the expected fields & values
that should be in the index
* @param actual A document that was returned by an RTG request
+ * @param wt the `wt` serialization of the response
* @return A set of "field names" in the actual document that this
validator expected.
*/
public Collection<String> assertRTGResults(final Collection<FlValidator>
validators,
final SolrInputDocument
expected,
- final SolrDocument actual);
+ final SolrDocument actual,
+ final String wt);
}
/**
@@ -554,15 +693,35 @@ public class TestRandomFlRTGCloud extends
SolrCloudTestCase {
this.actualFieldName = actualFieldName;
}
public abstract String getFlParam();
+
+ @Override
public Collection<String> assertRTGResults(final Collection<FlValidator>
validators,
final SolrInputDocument
expected,
- final SolrDocument actual) {
+ final SolrDocument actual,
+ final String wt) {
assertEquals(expectedFieldName + " vs " + actualFieldName,
- expected.getFieldValue(expectedFieldName),
actual.getFirstValue(actualFieldName));
+ expected.getFieldValue(expectedFieldName), normalize(wt,
actual.getFirstValue(actualFieldName)));
return Collections.<String>singleton(actualFieldName);
}
}
-
+
+ /**
+ * Json parsing results in all Long and Double number values; `expected`
values are all (conveniently!)
+ * expressed as Integer and Float, so we do a little normalization here so
that the values are compatible
+ */
+ private static Object normalize(String wt, Object val) {
+ if ("json".equals(wt) && val instanceof Number) {
+ if (val instanceof Long) {
+ return ((Long) val).intValue();
+ } else if (val instanceof Double) {
+ return ((Double) val).floatValue();
+ } else {
+ throw new IllegalStateException("numbers with `wt=json` only expect
Long or Double");
+ }
+ }
+ return val;
+ }
+
private static class SimpleFieldValueValidator extends FieldValueValidator {
public SimpleFieldValueValidator(final String fieldName) {
super(fieldName, fieldName);
@@ -588,15 +747,16 @@ public class TestRandomFlRTGCloud extends
SolrCloudTestCase {
* What we're primarily concerned with is that the transformer does it's job
and puts the string
* in the response, regardless of cloud/RTG/uncommited state of the document.
*/
- @SuppressWarnings("UnusedNestedClass") // SOLR-9376
private static class RawFieldValueValidator extends
RenameFieldValueValidator {
final String type;
final String alias;
+ final SolrParams extraParams;
public RawFieldValueValidator(final String type, final String fieldName,
final String alias) {
// transformer is weird, default result key doesn't care what params are
used...
super(fieldName, null == alias ? "["+type+"]" : alias);
this.type = type;
this.alias = alias;
+ this.extraParams = new ModifiableSolrParams().set(CommonParams.WT, type);
}
public RawFieldValueValidator(final String type, final String fieldName) {
this(type, fieldName, null);
@@ -604,11 +764,96 @@ public class TestRandomFlRTGCloud extends
SolrCloudTestCase {
public String getFlParam() {
return (null == alias ? "" : (alias + ":")) + "[" + type + " f=" +
expectedFieldName + "]";
}
+ @Override
+ public Collection<String> assertRTGResults(final Collection<FlValidator>
validators,
+ SolrInputDocument expected,
+ final SolrDocument actual,
+ final String wt) {
+ if ("json".equals(wt) && "json".equals(type)) {
+ Object v = actual.get(actualFieldName);
+ if (v instanceof Collection) {
+ // the json "array" type is indistinguishable from a multivalued
field, so when `super` validates
+ // based on `actual.getFirstValue(...)`, it causes issues. Here we
know that our raw values are only
+ // on single-valued fields, so we wrap it to work around
`getFirstValue` in parent class.
+ // The same logic applies to `expected` (below)
+ actual.setField(actualFieldName, Collections.singleton(v));
+ }
+ try {
+ Object parsedExpected = ObjectBuilder.fromJSON((String)
expected.getFieldValue(expectedFieldName));
+ if (parsedExpected instanceof Collection) {
+ // see note above
+ parsedExpected = Collections.singleton(parsedExpected);
+ }
+ expected = expected.deepCopy(); // need to copy before modifying
expected!
+ expected.setField(expectedFieldName, parsedExpected);
+ } catch (IOException ex) {
+ // swallow the exception and use the un-parsed String?
+ }
+ } else if ("xml".equals(wt) && "xml".equals(type)) {
+ try {
+ Object parsedExpected =
RawCapableXMLResponseParser.convertRawContent((String)
expected.getFieldValue(expectedFieldName));
+ expected = expected.deepCopy(); // need to copy before modifying
expected!
+ expected.setField(expectedFieldName, parsedExpected);
+ } catch (XMLStreamException ex) {
+ // swallow the exception and use the un-parsed String?
+ }
+ }
+ return super.assertRTGResults(validators, expected, actual, wt);
+ }
+ @Override
+ public SolrParams getExtraRequestParams() {
+ return extraParams;
+ }
public String getDefaultTransformerFactoryName() {
return type;
}
}
-
+
+ /**
+ * Local extension of XMLResponseParser that is capable of handling "raw"
xml field values, for the
+ * purpose of validation and consistency between expected vs. actual.
+ */
+ private static class RawCapableXMLResponseParser extends XMLResponseParser {
+
+ private static String convertRawContent(String raw) throws
XMLStreamException {
+ return XMLResponseParser.convertRawContent(raw, (parser) -> {
+ try {
+ return consumeRawContent0(parser);
+ } catch (XMLStreamException ex) {
+ // only called in the context of this test, so the extra exception
wrapping is totally fine
+ throw new RuntimeException(ex);
+ }
+ });
+ }
+
+ protected String consumeRawContent(XMLStreamReader parser) throws
XMLStreamException {
+ return consumeRawContent0(parser);
+ }
+
+ private static String consumeRawContent0(XMLStreamReader parser) throws
XMLStreamException {
+ int depth = 0;
+ StringBuilder sb = new StringBuilder();
+ for (;;) {
+ int elementType = parser.next();
+ switch (elementType) {
+ case XMLStreamConstants.START_ELEMENT:
+ depth++;
+ sb.append("START:").append(parser.getLocalName()).append(';');
+ break;
+ case XMLStreamConstants.END_ELEMENT:
+ if (--depth < 0) {
+ // exiting raw element
+ return sb.toString();
+ }
+ sb.append("END:").append(parser.getLocalName()).append(';');
+ break;
+ case XMLStreamConstants.CHARACTERS:
+ sb.append(parser.getText());
+ break;
+ }
+ }
+ }
+ }
/**
* enforces that a valid <code>[docid]</code> is present in the response,
possibly using a
@@ -631,8 +876,9 @@ public class TestRandomFlRTGCloud extends SolrCloudTestCase
{
public String getFlParam() { return USAGE.equals(resultKey) ? resultKey :
resultKey+":"+USAGE; }
public Collection<String> assertRTGResults(final Collection<FlValidator>
validators,
final SolrInputDocument
expected,
- final SolrDocument actual) {
- final Object value = actual.getFirstValue(resultKey);
+ final SolrDocument actual,
+ final String wt) {
+ Object value = normalize(wt, actual.getFirstValue(resultKey));
assertNotNull(getFlParam() + " => no value in actual doc", value);
assertTrue(USAGE + " must be an Integer: " + value, value instanceof
Integer);
@@ -664,7 +910,8 @@ public class TestRandomFlRTGCloud extends SolrCloudTestCase
{
public String getFlParam() { return USAGE.equals(resultKey) ? resultKey :
resultKey+":"+USAGE; }
public Collection<String> assertRTGResults(final Collection<FlValidator>
validators,
final SolrInputDocument
expected,
- final SolrDocument actual) {
+ final SolrDocument actual,
+ final String wt) {
final Object value = actual.getFirstValue(resultKey);
assertNotNull(getFlParam() + " => no value in actual doc", value);
assertTrue(USAGE + " must be an String: " + value, value instanceof
String);
@@ -699,8 +946,9 @@ public class TestRandomFlRTGCloud extends SolrCloudTestCase
{
public String getFlParam() { return fl; }
public Collection<String> assertRTGResults(final Collection<FlValidator>
validators,
final SolrInputDocument
expected,
- final SolrDocument actual) {
- final Object actualVal = actual.getFirstValue(resultKey);
+ final SolrDocument actual,
+ final String wt) {
+ Object actualVal = normalize(wt, actual.getFirstValue(resultKey));
assertNotNull(getFlParam() + " => no value in actual doc", actualVal);
assertEquals(getFlParam(), expectedVal, actualVal);
return Collections.<String>singleton(resultKey);
@@ -732,11 +980,12 @@ public class TestRandomFlRTGCloud extends
SolrCloudTestCase {
public String getFlParam() { return fl; }
public Collection<String> assertRTGResults(final Collection<FlValidator>
validators,
final SolrInputDocument
expected,
- final SolrDocument actual) {
+ final SolrDocument actual,
+ final String wt) {
final Object origVal = expected.getFieldValue(fieldName);
assertTrue("this validator only works on numeric fields: " + origVal,
origVal instanceof Number);
- assertEquals(fl, 1.3F, actual.getFirstValue(resultKey));
+ assertEquals(fl, 1.3F, normalize(wt, actual.getFirstValue(resultKey)));
return Collections.<String>singleton(resultKey);
}
}
@@ -764,12 +1013,17 @@ public class TestRandomFlRTGCloud extends
SolrCloudTestCase {
public final static String SUBQ_KEY = "subq";
public final static String SUBQ_FIELD = "next_2_ids_i";
public String getFlParam() { return SUBQ_KEY+":["+NAME+"]"; }
+ @SuppressWarnings("unchecked")
public Collection<String> assertRTGResults(final Collection<FlValidator>
validators,
final SolrInputDocument
expected,
- final SolrDocument actual) {
+ final SolrDocument actual,
+ final String wt) {
final int compVal = assertParseInt("expected id",
expected.getFieldValue("id"));
- final Object actualVal = actual.getFieldValue(SUBQ_KEY);
+ Object actualVal = actual.getFieldValue(SUBQ_KEY);
+ if ("json".equals(wt)) {
+ actualVal = getSolrDocumentList((Map<String, Object>) actualVal);
+ }
assertTrue("Expected a doclist: " + actualVal,
actualVal instanceof SolrDocumentList);
assertTrue("should be at most 2 docs in doc list: " + actualVal,
@@ -842,11 +1096,20 @@ public class TestRandomFlRTGCloud extends
SolrCloudTestCase {
public String getFlParam() { return fl; }
public Collection<String> assertRTGResults(final Collection<FlValidator>
validators,
final SolrInputDocument
expected,
- final SolrDocument actual) {
+ final SolrDocument actual,
+ final String wt) {
final Object origVal = expected.getFieldValue(fieldName);
assertTrue(fl + ": orig field value is not supported: " + origVal,
VALUES.containsKey(origVal));
-
- assertEquals(fl, VALUES.get(origVal), actual.getFirstValue(resultKey));
+
+ Object orig = VALUES.get(origVal);
+ if ("json".equals(wt)) {
+ try {
+ orig = ObjectBuilder.fromJSON((String) orig);
+ } catch (IOException ex) {
+ // swallow exception and use raw `orig` String?
+ }
+ }
+ assertEquals(fl, orig, actual.getFirstValue(resultKey));
return Collections.<String>singleton(resultKey);
}
public Set<String> getSuppressedFields() { return
Collections.singleton(fieldName); }
@@ -881,7 +1144,8 @@ public class TestRandomFlRTGCloud extends
SolrCloudTestCase {
public Collection<String> assertRTGResults(final Collection<FlValidator>
validators,
final SolrInputDocument
expected,
- final SolrDocument actual) {
+ final SolrDocument actual,
+ final String wt) {
final Set<String> renamed = new LinkedHashSet<>(validators.size());
for (FlValidator v : validators) {
@@ -895,7 +1159,7 @@ public class TestRandomFlRTGCloud extends
SolrCloudTestCase {
for (String f : expected.getFieldNames()) {
if ( matchesGlob(f) && (! renamed.contains(f) ) ) {
result.add(f);
- assertEquals(glob + " => " + f, expected.getFieldValue(f),
actual.getFirstValue(f));
+ assertEquals(glob + " => " + f, expected.getFieldValue(f),
normalize(wt, actual.getFirstValue(f)));
}
}
return result;
@@ -919,7 +1183,8 @@ public class TestRandomFlRTGCloud extends
SolrCloudTestCase {
public String getFlParam() { return fl; }
public Collection<String> assertRTGResults(final Collection<FlValidator>
validators,
final SolrInputDocument
expected,
- final SolrDocument actual) {
+ final SolrDocument actual,
+ final String wt) {
assertEquals(fl, null, actual.getFirstValue(fieldName));
return Collections.emptySet();
}
diff --git a/solr/core/src/test/org/apache/solr/response/JSONWriterTest.java
b/solr/core/src/test/org/apache/solr/response/JSONWriterTest.java
index ea0b3d3..a5a3396 100644
--- a/solr/core/src/test/org/apache/solr/response/JSONWriterTest.java
+++ b/solr/core/src/test/org/apache/solr/response/JSONWriterTest.java
@@ -203,13 +203,13 @@ public class JSONWriterTest extends SolrTestCaseJ4 {
methodsExpectedNotOverriden.add("writeMapOpener");
methodsExpectedNotOverriden.add("writeMapSeparator");
methodsExpectedNotOverriden.add("writeMapCloser");
- methodsExpectedNotOverriden.add("public default void
org.apache.solr.common.util.JsonTextWriter.writeArray(java.lang.String,java.util.List)
throws java.io.IOException");
+ methodsExpectedNotOverriden.add("public default void
org.apache.solr.common.util.JsonTextWriter.writeArray(java.lang.String,java.util.List,boolean)
throws java.io.IOException");
methodsExpectedNotOverriden.add("writeArrayOpener");
methodsExpectedNotOverriden.add("writeArraySeparator");
methodsExpectedNotOverriden.add("writeArrayCloser");
methodsExpectedNotOverriden.add("public default void
org.apache.solr.common.util.JsonTextWriter.writeMap(org.apache.solr.common.MapWriter)
throws java.io.IOException");
methodsExpectedNotOverriden.add("public default void
org.apache.solr.common.util.JsonTextWriter.writeIterator(org.apache.solr.common.IteratorWriter)
throws java.io.IOException");
- methodsExpectedNotOverriden.add("public default void
org.apache.solr.common.util.JsonTextWriter.writeJsonIter(java.util.Iterator)
throws java.io.IOException");
+ methodsExpectedNotOverriden.add("public default void
org.apache.solr.common.util.JsonTextWriter.writeJsonIter(java.util.Iterator,boolean)
throws java.io.IOException");
final Class<?> subClass =
JSONResponseWriter.ArrayOfNameTypeValueJSONWriter.class;
final Class<?> superClass = subClass.getSuperclass();
diff --git
a/solr/core/src/test/org/apache/solr/response/TestRawTransformer.java
b/solr/core/src/test/org/apache/solr/response/TestRawTransformer.java
index ea5619f..28fa1c2 100644
--- a/solr/core/src/test/org/apache/solr/response/TestRawTransformer.java
+++ b/solr/core/src/test/org/apache/solr/response/TestRawTransformer.java
@@ -16,55 +16,201 @@
*/
package org.apache.solr.response;
+import org.apache.commons.io.FileUtils;
import org.apache.solr.SolrTestCaseJ4;
+import org.apache.solr.client.solrj.embedded.JettySolrRunner;
+import org.apache.solr.client.solrj.impl.CloudSolrClient;
+import org.apache.solr.client.solrj.impl.HttpSolrClient;
+import org.apache.solr.client.solrj.impl.NoOpResponseParser;
+import org.apache.solr.client.solrj.request.CollectionAdminRequest;
+import org.apache.solr.client.solrj.request.QueryRequest;
+import org.apache.solr.cloud.MiniSolrCloudCluster;
+import org.apache.solr.cloud.SolrCloudTestCase;
import org.apache.solr.common.SolrInputDocument;
-import org.apache.solr.request.SolrQueryRequest;
+import org.apache.solr.common.params.ModifiableSolrParams;
import org.junit.After;
+import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
+import java.io.File;
+import java.lang.invoke.MethodHandles;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.Properties;
+import java.util.regex.Pattern;
+
/**
* Tests Raw JSON output for fields when used with and without the unique key
field.
*
* See SOLR-7993
*/
-public class TestRawTransformer extends SolrTestCaseJ4 {
+public class TestRawTransformer extends SolrCloudTestCase {
+
+ private static final String DEBUG_LABEL =
MethodHandles.lookup().lookupClass().getName();
+
+ /** A basic client for operations at the cloud level, default collection
will be set */
+ private static JettySolrRunner JSR;
+ private static HttpSolrClient CLIENT;
@BeforeClass
public static void beforeClass() throws Exception {
- initCore("solrconfig-doctransformers.xml", "schema.xml");
+ if (random().nextBoolean()) {
+ initStandalone();
+ JSR.start();
+ CLIENT = (HttpSolrClient) JSR.newClient();
+ } else {
+ initCloud();
+ CLIENT = (HttpSolrClient) JSR.newClient();
+ JSR = null;
+ }
+ initIndex();
+ }
+
+ private static void initStandalone() throws Exception {
+ initCore("solrconfig-minimal.xml", "schema_latest.xml");
+ File homeDir = createTempDir().toFile();
+ final File collDir = new File(homeDir, "collection1");
+ final File confDir = collDir.toPath().resolve("conf").toFile();
+ confDir.mkdirs();
+ FileUtils.copyFile(new File(SolrTestCaseJ4.TEST_HOME(), "solr.xml"), new
File(homeDir, "solr.xml"));
+ String src_dir = TEST_HOME() + "/collection1/conf";
+ FileUtils.copyFile(new File(src_dir, "schema_latest.xml"),
+ new File(confDir, "schema.xml"));
+ FileUtils.copyFile(new File(src_dir, "solrconfig-minimal.xml"),
+ new File(confDir, "solrconfig.xml"));
+ for (String file : new String[]
{"solrconfig.snippet.randomindexconfig.xml",
+ "stopwords.txt", "synonyms.txt", "protwords.txt", "currency.xml"})
{
+ FileUtils.copyFile(new File(src_dir, file), new File(confDir, file));
+ }
+ Files.createFile(collDir.toPath().resolve("core.properties"));
+ Properties nodeProperties = new Properties();
+ nodeProperties.setProperty("solr.data.dir", h.getCore().getDataDir());
+ JSR = new JettySolrRunner(homeDir.getAbsolutePath(), nodeProperties,
buildJettyConfig("/solr"));
+ }
+
+ private static void initCloud() throws Exception {
+ final String configName = DEBUG_LABEL + "_config-set";
+ final Path configDir = Paths.get(TEST_HOME(), "collection1", "conf");
+
+ final int numNodes = 3;
+ MiniSolrCloudCluster cloud =
configureCluster(numNodes).addConfig(configName, configDir).configure();
+
+ Map<String, String> collectionProperties = new LinkedHashMap<>();
+ collectionProperties.put("config", "solrconfig-minimal.xml");
+ collectionProperties.put("schema", "schema_latest.xml");
+ CloudSolrClient cloudSolrClient = cloud.getSolrClient();
+ CollectionAdminRequest.createCollection("collection1", configName,
numNodes, 1)
+ .setPerReplicaState(SolrCloudTestCase.USE_PER_REPLICA_STATE)
+ .setProperties(collectionProperties)
+ .process(cloudSolrClient);
+
+ JSR = cloud.getRandomJetty(random());
+ }
+
+ @AfterClass
+ private static void afterClass() throws Exception{
+ if (JSR != null) {
+ JSR.stop();
+ }
+ // NOTE: CLOUD_CLIENT should be stopped automatically in
`SolrCloudTestCase.shutdownCluster()`
}
@After
public void cleanup() throws Exception {
- assertU(delQ("*:*"));
- assertU(commit());
+ if (JSR != null) {
+ assertU(delQ("*:*"));
+ assertU(commit());
+ }
}
- @Test
- public void testCustomTransformer() throws Exception {
+ private static final int MAX = 10;
+ private static void initIndex() throws Exception {
// Build a simple index
- int max = 10;
- for (int i = 0; i < max; i++) {
+ // TODO: why are we indexing 10 docs here? Wouldn't one suffice?
+ for (int i = 0; i < MAX; i++) {
SolrInputDocument sdoc = new SolrInputDocument();
sdoc.addField("id", i);
+ // below are single-valued fields
sdoc.addField("subject",
"{poffL:[{offL:[{oGUID:\"79D5A31D-B3E4-4667-B812-09DF4336B900\",oID:\"OO73XRX\",prmryO:1,oRank:1,addTp:\"Office\",addCd:\"AA4GJ5T\",ad1:\"102
S 3rd St Ste 100\",city:\"Carson
City\",st:\"MI\",zip:\"48811\",lat:43.176885,lng:-84.842919,phL:[\"(989)
584-1308\"],faxL:[\"(989) 584-6453\"]}]}]}");
- sdoc.addField("title", "title_" + i);
- updateJ(jsonAdd(sdoc), null);
+ sdoc.addField("author",
"<root><child1>some</child1><child2>trivial</child2><child3>xml</child3></root>");
+ // below are multiValued fields
+ sdoc.addField("links", "{an_array:[1,2,3]}");
+ sdoc.addField("links", "{an_array:[4,5,6]}");
+ sdoc.addField("content_type", "<root>one</root>");
+ sdoc.addField("content_type", "<root>two</root>");
+ CLIENT.add("collection1", sdoc);
}
- assertU(commit());
- assertQ(req("q", "*:*"), "//*[@numFound='" + max + "']");
+ CLIENT.commit("collection1");
+ assertEquals(MAX, CLIENT.query("collection1", new
ModifiableSolrParams(Map.of("q", new
String[]{"*:*"}))).getResults().getNumFound());
+ }
+
+ @Test
+ public void testXmlTransformer() throws Exception {
+ QueryRequest req = new QueryRequest(new ModifiableSolrParams(
+ Map.of("q", new String[]{"*:*"}, "fl", new
String[]{"author:[xml],content_type:[xml]"}, "wt", new String[]{"xml"})
+ ));
+ req.setResponseParser(XML_NOOP_RESPONSE_PARSER);
+ String strResponse = (String)
CLIENT.request(req,"collection1").get("response");
+ assertTrue("response does not contain raw XML encoding: " + strResponse,
+ strResponse.contains("<raw
name=\"author\"><root><child1>some</child1><child2>trivial</child2><child3>xml</child3></root></raw>"));
+ assertTrue("response (multiValued) does not contain raw XML encoding: " +
strResponse,
+ Pattern.compile("<arr
name=\"content_type\">\\s*<raw><root>one</root></raw>\\s*<raw><root>two</root></raw>\\s*</arr>").matcher(strResponse).find());
- SolrQueryRequest req = req("q", "*:*", "fl", "subject:[json]", "wt",
"json");
- String strResponse = h.query(req);
+ req = new QueryRequest(new ModifiableSolrParams(
+ Map.of("q", new String[]{"*:*"}, "fl", new
String[]{"author,content_type"}, "wt", new String[]{"xml"})
+ ));
+ req.setResponseParser(XML_NOOP_RESPONSE_PARSER);
+ strResponse = (String) CLIENT.request(req, "collection1").get("response");
+ assertTrue("response does not contain escaped XML encoding: " +
strResponse,
+ strResponse.contains("<str
name=\"author\"><root><child1"));
+ assertTrue("response (multiValued) does not contain escaped XML encoding:
" + strResponse,
+ Pattern.compile("<arr
name=\"content_type\">\\s*<str><root>").matcher(strResponse).find());
+
+ req = new QueryRequest(new ModifiableSolrParams(
+ Map.of("q", new String[]{"*:*"}, "fl", new
String[]{"author:[xml],content_type:[xml]"}, "wt", new String[]{"json"})
+ ));
+ req.setResponseParser(JSON_NOOP_RESPONSE_PARSER);
+ strResponse = (String) CLIENT.request(req, "collection1").get("response");
+ assertTrue("unexpected serialization of XML field value in JSON response:
" + strResponse,
+ strResponse.contains("\"author\":\"<root><child1>some</child1>"));
+ assertTrue("unexpected (multiValued) serialization of XML field value in
JSON response: " + strResponse,
+ strResponse.contains("\"content_type\":[\"<root>one</root>"));
+ }
+
+ @Test
+ public void testJsonTransformer() throws Exception {
+ QueryRequest req = new QueryRequest(new ModifiableSolrParams(
+ Map.of("q", new String[]{"*:*"}, "fl", new
String[]{"subject:[json],links:[json]"}, "wt", new String[]{"json"})
+ ));
+ req.setResponseParser(JSON_NOOP_RESPONSE_PARSER);
+ String strResponse = (String)
CLIENT.request(req,"collection1").get("response");
assertTrue("response does not contain right JSON encoding: " + strResponse,
- strResponse.contains("\"subject\":[{poffL:[{offL:[{oGUID:\"7"));
+ strResponse.contains("\"subject\":{poffL:[{offL:[{oGUID:\"7"));
+ assertTrue("response (multiValued) does not contain right JSON encoding: "
+ strResponse,
+
Pattern.compile("\"links\":\\[\\{an_array:\\[1,2,3]},\\s*\\{an_array:\\[4,5,6]}]").matcher(strResponse).find());
- req = req("q", "*:*", "fl", "id,subject", "wt", "json");
- strResponse = h.query(req);
+ req = new QueryRequest(new ModifiableSolrParams(
+ Map.of("q", new String[]{"*:*"}, "fl", new String[]{"id",
"subject,links"}, "wt", new String[]{"json"})
+ ));
+ req.setResponseParser(JSON_NOOP_RESPONSE_PARSER);
+ strResponse = (String) CLIENT.request(req, "collection1").get("response");
assertTrue("response does not contain right JSON encoding: " + strResponse,
- strResponse.contains("subject\":[\""));
+ strResponse.contains("subject\":\""));
+ assertTrue("response (multiValued) does not contain right JSON encoding: "
+ strResponse,
+ strResponse.contains("\"links\":[\""));
}
+ private static final NoOpResponseParser XML_NOOP_RESPONSE_PARSER = new
NoOpResponseParser();
+ private static final NoOpResponseParser JSON_NOOP_RESPONSE_PARSER = new
NoOpResponseParser() {
+ @Override
+ public String getWriterType() {
+ return "json";
+ }
+ };
+
}
diff --git
a/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/XLSXResponseWriter.java
b/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/XLSXResponseWriter.java
index b225ceb..d1e5dda 100644
---
a/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/XLSXResponseWriter.java
+++
b/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/XLSXResponseWriter.java
@@ -258,7 +258,7 @@ class XLSXWriter extends TabularResponseWriter {
values = tmpList;
}
- writeArray(xlField.name, values.iterator());
+ writeArray(xlField.name, values.iterator(), false);
} else {
// normalize to first value
@@ -278,7 +278,8 @@ class XLSXWriter extends TabularResponseWriter {
}
@Override
- public void writeArray(String name, Iterator<?> val) throws IOException {
+ public void writeArray(String name, Iterator<?> val, boolean raw) throws
IOException {
+ assert !raw;
StringBuffer output = new StringBuffer();
while (val.hasNext()) {
Object v = val.next();
diff --git a/solr/solr-ref-guide/src/major-changes-in-solr-9.adoc
b/solr/solr-ref-guide/src/major-changes-in-solr-9.adoc
index 27eb361..6d0b109 100644
--- a/solr/solr-ref-guide/src/major-changes-in-solr-9.adoc
+++ b/solr/solr-ref-guide/src/major-changes-in-solr-9.adoc
@@ -152,6 +152,13 @@ Currently this change should only effect compatibility of
custom code overriding
* SOLR-14510: The `writeStartDocumentList` in `TextResponseWriter` now
receives an extra boolean parameter representing the "exactness" of the
`numFound` value (exact vs approximation).
Any custom response writer extending `TextResponseWriter` will need to
implement this abstract method now (instead previous with the same name but
without the new boolean parameter).
+* SOLR-9376: The response format for field values serialized as raw XML (via
the `[xml]` raw value DocTransformer
+and `wt=xml`) has changed. Previously, values were dropped in directly as
top-level child elements of each `<doc>`,
+obscuring associated field names and yielding inconsistent `<doc>` structure.
As of version 9.0, raw values are
+wrapped in a `<raw name="field_name">[...]</raw>` element at the top level of
each `<doc>` (or within an enclosing
+`<arr name="field_name"><raw>[...]</raw></arr>` element for multi-valued
fields). Existing clients that parse field
+values serialized in this way will need to be updated accordingly.
+
=== solr.xml maxBooleanClauses now enforced recursively
Lucene 9.0 has additional safety checks over previous versions that impact how
the `solr.xml` global
`<<configuring-solr-xml#global-maxbooleanclauses,maxBooleanClauses>>` option is
enforced.
diff --git
a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/XMLResponseParser.java
b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/XMLResponseParser.java
index 40cc649..c219e70 100644
---
a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/XMLResponseParser.java
+++
b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/XMLResponseParser.java
@@ -22,12 +22,14 @@ import javax.xml.stream.XMLStreamException;
import javax.xml.stream.XMLStreamReader;
import java.io.InputStream;
import java.io.Reader;
+import java.io.StringReader;
import java.lang.invoke.MethodHandles;
import java.time.Instant;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.Locale;
+import java.util.function.Function;
import org.apache.solr.client.solrj.ResponseParser;
import org.apache.solr.common.EmptyEntityResolver;
@@ -183,6 +185,7 @@ public class XMLResponseParser extends ResponseParser
}
},
+ RAW (true) { @Override public Object read( String txt ) { return null;
} },
ARR (false) { @Override public Object read( String txt ) { return null;
} },
LST (false) { @Override public Object read( String txt ) { return null;
} },
RESULT (false) { @Override public Object read( String txt ) { return null;
} },
@@ -262,6 +265,7 @@ public class XMLResponseParser extends ResponseParser
case LONG:
case NULL:
case STR:
+ case RAW:
break;
}
throw new XMLStreamException( "branch element not handled!",
parser.getLocation() );
@@ -335,6 +339,7 @@ public class XMLResponseParser extends ResponseParser
case LONG:
case NULL:
case STR:
+ case RAW:
break;
}
throw new XMLStreamException( "branch element not handled!",
parser.getLocation() );
@@ -454,6 +459,15 @@ public class XMLResponseParser extends ResponseParser
} else if( type == KnownType.LST ) {
doc.addField( name, readNamedList( parser ) );
depth--;
+ } else if( type == KnownType.RESULT ) {
+ // e.g., from the [subquery] doc transformer
+ doc.put(name, readDocuments(parser));
+ depth--;
+ } else if( type == KnownType.RAW ) {
+ // e.g., from the raw [xml] doc transformer.
+ String raw = consumeRawContent(parser);
+ doc.addField(name, raw);
+ depth--;
} else if( !type.isLeaf ) {
throw new XMLStreamException( "must be value or array",
parser.getLocation() );
}
@@ -480,5 +494,29 @@ public class XMLResponseParser extends ResponseParser
}
}
+ /**
+ * This is a stub method for handling/validating "raw" xml field values in
the context of tests. Before this
+ * stub method was present, "raw" content would have still thrown an
exception, albeit a different, more inscrutable
+ * exception.
+ */
+ protected String consumeRawContent(XMLStreamReader parser) throws
XMLStreamException {
+ throw new UnsupportedOperationException(XMLResponseParser.class + " is not
capable of consuming field values serialized as raw XML");
+ }
+
+ /**
+ * Convenience method that converts raw String input (should be valid xml
when wrapped in a root element) and
+ * converts it to a format compatible with how {@link XMLResponseParser}
parses from raw xml fields.
+ * This method is intended for test validation.
+ *
+ * The main reason this method exists is to provide a consistent way of
configuring and creating and invoking
+ * the sub-parser
+ */
+ protected static String convertRawContent(String raw,
Function<XMLStreamReader, String> consumeRawContent) throws XMLStreamException {
+ XMLStreamReader subParser = factory.createXMLStreamReader(new
StringReader("<raw>" + raw + "</raw>"));
+ while (subParser.next() != XMLStreamConstants.START_ELEMENT) {
+ // consume any early stuff
+ }
+ return consumeRawContent.apply(subParser);
+ }
}
diff --git
a/solr/solrj/src/java/org/apache/solr/common/util/JsonTextWriter.java
b/solr/solrj/src/java/org/apache/solr/common/util/JsonTextWriter.java
index 42025e3..85de57b 100644
--- a/solr/solrj/src/java/org/apache/solr/common/util/JsonTextWriter.java
+++ b/solr/solrj/src/java/org/apache/solr/common/util/JsonTextWriter.java
@@ -65,6 +65,10 @@ public interface JsonTextWriter extends TextWriter {
_writeChar(']');
}
+ default void writeStrRaw(String name, String val) throws IOException {
+ _writeStr(val);
+ }
+
default void writeStr(String name, String val, boolean needsEscaping) throws
IOException {
// it might be more efficient to use a stringbuilder or write substrings
// if writing chars to the stream is slow.
@@ -185,12 +189,12 @@ public interface JsonTextWriter extends TextWriter {
_writeChar(':');
}
- default void writeJsonIter(Iterator<?> val) throws IOException {
+ default void writeJsonIter(Iterator<?> val, boolean raw) throws IOException {
incLevel();
boolean first = true;
while (val.hasNext()) {
if (!first) indent();
- writeVal(null, val.next());
+ writeVal(null, val.next(), raw);
if (val.hasNext()) {
writeArraySeparator();
}
@@ -264,15 +268,15 @@ public interface JsonTextWriter extends TextWriter {
}
- default void writeArray(String name, List<?> l) throws IOException {
+ default void writeArray(String name, List<?> l, boolean raw) throws
IOException {
writeArrayOpener(l.size());
- writeJsonIter(l.iterator());
+ writeJsonIter(l.iterator(), raw);
writeArrayCloser();
}
- default void writeArray(String name, Iterator<?> val) throws IOException {
+ default void writeArray(String name, Iterator<?> val, boolean raw) throws
IOException {
writeArrayOpener(-1); // no trivial way to determine array size
- writeJsonIter(val);
+ writeJsonIter(val, raw);
writeArrayCloser();
}
diff --git a/solr/solrj/src/java/org/apache/solr/common/util/TextWriter.java
b/solr/solrj/src/java/org/apache/solr/common/util/TextWriter.java
index e02677f..a7605f1 100644
--- a/solr/solrj/src/java/org/apache/solr/common/util/TextWriter.java
+++ b/solr/solrj/src/java/org/apache/solr/common/util/TextWriter.java
@@ -43,6 +43,10 @@ import org.apache.solr.common.PushWriter;
public interface TextWriter extends PushWriter {
default void writeVal(String name, Object val) throws IOException {
+ writeVal(name, val, false);
+ }
+
+ default void writeVal(String name, Object val, boolean raw) throws
IOException {
// if there get to be enough types, perhaps hashing on the type
// to get a handler might be faster (but types must be exact to do that...)
@@ -52,8 +56,12 @@ public interface TextWriter extends PushWriter {
if (val == null) {
writeNull(name);
} else if (val instanceof CharSequence) {
- writeStr(name, val.toString(), true);
- // micro-optimization... using toString() avoids a cast first
+ if (raw) {
+ writeStrRaw(name, val.toString());
+ } else {
+ writeStr(name, val.toString(), true);
+ // micro-optimization... using toString() avoids a cast first
+ }
} else if (val instanceof Number) {
writeNumber(name, (Number) val);
} else if (val instanceof Boolean) {
@@ -65,9 +73,14 @@ public interface TextWriter extends PushWriter {
} else if (val instanceof NamedList) {
writeNamedList(name, (NamedList)val);
} else if (val instanceof Path) {
- writeStr(name, ((Path) val).toAbsolutePath().toString(), true);
+ final String pathStr = ((Path) val).toAbsolutePath().toString();
+ if (raw) {
+ writeStrRaw(name, pathStr);
+ } else {
+ writeStr(name, pathStr, true);
+ }
} else if (val instanceof IteratorWriter) {
- writeIterator(name, (IteratorWriter) val);
+ writeIterator(name, (IteratorWriter) val, raw);
} else if (val instanceof MapWriter) {
writeMap(name, (MapWriter) val);
} else if (val instanceof MapSerializable) {
@@ -76,29 +89,39 @@ public interface TextWriter extends PushWriter {
} else if (val instanceof Map) {
writeMap(name, (Map)val, false, true);
} else if (val instanceof Iterator) { // very generic; keep towards the end
- writeArray(name, (Iterator) val);
+ writeArray(name, (Iterator) val, raw);
} else if (val instanceof Iterable) { // very generic; keep towards the end
- writeArray(name,((Iterable)val).iterator());
+ writeArray(name,((Iterable)val).iterator(), raw);
} else if (val instanceof Object[]) {
- writeArray(name,(Object[])val);
+ writeArray(name,(Object[])val, raw);
} else if (val instanceof byte[]) {
byte[] arr = (byte[])val;
writeByteArr(name, arr, 0, arr.length);
} else if (val instanceof EnumFieldValue) {
- writeStr(name, val.toString(), true);
- } else if (val instanceof WriteableValue) {
- ((WriteableValue)val).write(name, this);
+ if (raw) {
+ writeStrRaw(name, val.toString());
+ } else {
+ writeStr(name, val.toString(), true);
+ }
} else {
// default... for debugging only. Would be nice to "assert false" ?
writeStr(name, val.getClass().getName() + ':' + val.toString(), true);
}
}
+ /**
+ * Writes the specified val directly to the backing writer, without wrapping
(e.g., in quotes) or escaping
+ * of any kind.
+ */
+ default void writeStrRaw(String name, String val) throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
void writeStr(String name, String val, boolean needsEscaping) throws
IOException;
void writeMap(String name, Map<?, ?> val, boolean excludeOuter, boolean
isFirstVal) throws IOException;
- void writeArray(String name, Iterator<?> val) throws IOException;
+ void writeArray(String name, Iterator<?> val, boolean raw) throws
IOException;
void writeNull(String name) throws IOException;
@@ -153,12 +176,12 @@ public interface TextWriter extends PushWriter {
}
}
- default void writeArray(String name, Object[] val) throws IOException {
- writeArray(name, Arrays.asList(val));
+ default void writeArray(String name, Object[] val, boolean raw) throws
IOException {
+ writeArray(name, Arrays.asList(val), raw);
}
- default void writeArray(String name, List<?> l) throws IOException {
- writeArray(name, l.iterator());
+ default void writeArray(String name, List<?> l, boolean raw) throws
IOException {
+ writeArray(name, l.iterator(), raw);
}
@@ -224,7 +247,7 @@ public interface TextWriter extends PushWriter {
/*todo*/
}
- default void writeIterator(String name, IteratorWriter iw) throws
IOException {
+ default void writeIterator(String name, IteratorWriter iw, boolean raw)
throws IOException {
writeIterator(iw);
}
diff --git
a/solr/solrj/src/java/org/apache/solr/common/util/WriteableValue.java
b/solr/solrj/src/java/org/apache/solr/common/util/WriteableValue.java
deleted file mode 100644
index 82e7d00..0000000
--- a/solr/solrj/src/java/org/apache/solr/common/util/WriteableValue.java
+++ /dev/null
@@ -1,25 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.common.util;
-
-import java.io.IOException;
-
-import org.apache.solr.common.util.JavaBinCodec.ObjectResolver;
-
-public abstract class WriteableValue implements ObjectResolver {
- public abstract void write(String name, TextWriter writer) throws
IOException;
-}
\ No newline at end of file
