This is an automated email from the ASF dual-hosted git repository.
hossman pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/solr.git
The following commit(s) were added to refs/heads/main by this push:
new 90ef6ff725d SOLR-18033: Solr response writing now correctly delegates
to FieldType implementations to determine how internal binary field values are
represented externally
90ef6ff725d is described below
commit 90ef6ff725db6f7c530d6a8183216b07b75af38f
Author: Chris Hostetter <[email protected]>
AuthorDate: Tue Jan 13 11:10:56 2026 -0700
SOLR-18033: Solr response writing now correctly delegates to FieldType
implementations to determine how internal binary field values are represented
externally
---
...d-fieldtype-external-representation-control.yml | 7 ++
.../org/apache/solr/response/DocsStreamer.java | 28 ++++-
.../java/org/apache/solr/schema/BinaryField.java | 5 +
.../src/java/org/apache/solr/schema/FieldType.java | 24 ++++
.../apache/solr/search/SolrDocumentFetcher.java | 2 +-
.../solr/collection1/conf/schema-binaryfield.xml | 10 +-
.../conf/schema-non-stored-docvalues.xml | 15 +++
.../org/apache/solr/schema/StrBinaryField.java | 88 ++++++++++++++
.../apache/solr/schema/SwapBytesBinaryField.java | 88 ++++++++++++++
.../org/apache/solr/schema/TestBinaryField.java | 130 ++++++++++++--------
.../solr/schema/TestUseDocValuesAsStored.java | 134 ++++++++++++++++++++-
11 files changed, 473 insertions(+), 58 deletions(-)
diff --git
a/changelog/unreleased/SOLR-18033-binary-based-fieldtype-external-representation-control.yml
b/changelog/unreleased/SOLR-18033-binary-based-fieldtype-external-representation-control.yml
new file mode 100644
index 00000000000..2cfcfd34cb1
--- /dev/null
+++
b/changelog/unreleased/SOLR-18033-binary-based-fieldtype-external-representation-control.yml
@@ -0,0 +1,7 @@
+title: Solr response writing now correctly delegates to FieldType
implementations to determine how internal binary field values are represented
externally
+type: fixed
+authors:
+- name: hossman
+links:
+- name: SOLR-18033
+ url: https://issues.apache.org/jira/browse/SOLR-18033
diff --git a/solr/core/src/java/org/apache/solr/response/DocsStreamer.java
b/solr/core/src/java/org/apache/solr/response/DocsStreamer.java
index e980e809b18..850781c079d 100644
--- a/solr/core/src/java/org/apache/solr/response/DocsStreamer.java
+++ b/solr/core/src/java/org/apache/solr/response/DocsStreamer.java
@@ -16,6 +16,8 @@
*/
package org.apache.solr.response;
+import static
org.apache.solr.schema.FieldType.ExternalizeStoredValuesAsObjects;
+
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashSet;
@@ -55,7 +57,22 @@ import org.apache.solr.search.SolrReturnFields;
/** This streams SolrDocuments from a DocList and applies transformer */
public class DocsStreamer implements Iterator<SolrDocument> {
- public static final Set<Class<? extends FieldType>> KNOWN_TYPES = new
HashSet<>();
+ /**
+ * A hardcoded list of known Solr field types that will be trusted to
control their own conversion
+ * of stored field values into external Objects (via {@link
FieldType#toObject}) when returning
+ * {@link SolrDocument} instances to clients.
+ *
+ * <p>For historic reasons, this Set is consulted using an <em>equality</em>
basis, so subclasses
+ * of these "known" types are not given the same level of trust.
+ *
+ * <p>Any field type not found in this list will have stored values
externalized as
+ * <em>Strings</em> using {@link FieldType#toExternal} unless they implement
{@link
+ * ExternalizeStoredValuesAsObjects}
+ *
+ * @deprecated new field types should not be added to this list, instead use
{@link
+ * ExternalizeStoredValuesAsObjects}
+ */
+ @Deprecated public static final Set<Class<? extends FieldType>> KNOWN_TYPES
= new HashSet<>();
private final org.apache.solr.response.ResultContext rctx;
private final SolrDocumentFetcher docFetcher; // a collaborator of
SolrIndexSearcher
@@ -200,7 +217,8 @@ public class DocsStreamer implements Iterator<SolrDocument>
{
return f.stringValue();
}
} else {
- if (KNOWN_TYPES.contains(ft.getClass())) {
+ if (KNOWN_TYPES.contains(ft.getClass())
+ || ft instanceof FieldType.ExternalizeStoredValuesAsObjects) {
return ft.toObject(f);
} else {
return ft.toExternal(f);
@@ -209,6 +227,9 @@ public class DocsStreamer implements Iterator<SolrDocument>
{
}
static {
+ // DO NOT ADD TO THIS SET ! ! ! !
+ // SEE JAVADOCS FOR KNOWN_TYPES !
+
KNOWN_TYPES.add(BoolField.class);
KNOWN_TYPES.add(StrField.class);
KNOWN_TYPES.add(TextField.class);
@@ -229,5 +250,8 @@ public class DocsStreamer implements Iterator<SolrDocument>
{
KNOWN_TYPES.add(DatePointField.class);
// We do not add UUIDField because UUID object is not a supported type in
JavaBinCodec
// and if we write UUIDField.toObject, we wouldn't know how to handle it
in the client side
+
+ // DO NOT ADD TO THIS SET ! ! ! !
+ // SEE JAVADOCS FOR KNOWN_TYPES !
}
}
diff --git a/solr/core/src/java/org/apache/solr/schema/BinaryField.java
b/solr/core/src/java/org/apache/solr/schema/BinaryField.java
index 1659e51f870..db05597d75f 100644
--- a/solr/core/src/java/org/apache/solr/schema/BinaryField.java
+++ b/solr/core/src/java/org/apache/solr/schema/BinaryField.java
@@ -96,6 +96,11 @@ public class BinaryField extends FieldType {
return ByteBuffer.wrap(bytes.bytes, bytes.offset, bytes.length);
}
+ @Override
+ public Object toObject(SchemaField sf, BytesRef term) {
+ return BytesRef.deepCopyOf(term).bytes;
+ }
+
@Override
public IndexableField createField(SchemaField field, Object val) {
if (val == null) return null;
diff --git a/solr/core/src/java/org/apache/solr/schema/FieldType.java
b/solr/core/src/java/org/apache/solr/schema/FieldType.java
index 2f922473e50..15387038bba 100644
--- a/solr/core/src/java/org/apache/solr/schema/FieldType.java
+++ b/solr/core/src/java/org/apache/solr/schema/FieldType.java
@@ -366,6 +366,10 @@ public abstract class FieldType extends FieldProperties {
/**
* Convert the stored-field format to an external (string, human readable)
value
*
+ * <p>This is the default method used for converting a stored field value
into an external value
+ * to be returned to clients. See {@link ExternalizeStoredValuesAsObjects}
for more details
+ *
+ * @see #toObject(IndexableField)
* @see #toInternal
*/
public String toExternal(IndexableField f) {
@@ -383,6 +387,10 @@ public abstract class FieldType extends FieldProperties {
/**
* Convert the stored-field format to an external object.
*
+ * <p>This method is not typically used for custom FieldTypes, see {@link
+ * ExternalizeStoredValuesAsObjects} for more details
+ *
+ * @see #toExternal
* @see #toInternal
* @since solr 1.3
*/
@@ -1460,6 +1468,22 @@ public abstract class FieldType extends FieldProperties {
return new BytesRef(bytes);
}
+ /**
+ * A marker interface that can be implemented by any FieldType to indicate
that Solr should trust
+ * & delegate to this field type's implementation of {@link
+ * FieldType#toObject(IndexableField)} when converted internal stored fields
to an external
+ * representation that will be returned to clients.
+ *
+ * <p>The default behavior if this interface is not implemented, is to
delegate to {@link
+ * FieldType#toExternal(IndexableField)}, unless the field type is (exactly
equal to) one of a
+ * specific list of {@link org.apache.solr.response.DocsStreamer#KNOWN_TYPES}
+ *
+ * @see #toExternal
+ * @see #toObject(IndexableField)
+ * @see org.apache.solr.response.DocsStreamer#KNOWN_TYPES
+ */
+ public static interface ExternalizeStoredValuesAsObjects {}
+
/**
* An enumeration representing various options that may exist for selecting
a single value from a
* multivalued field. This class is designed to be an abstract
representation, agnostic of some of
diff --git a/solr/core/src/java/org/apache/solr/search/SolrDocumentFetcher.java
b/solr/core/src/java/org/apache/solr/search/SolrDocumentFetcher.java
index d1a3f3ddef9..6c31193014b 100644
--- a/solr/core/src/java/org/apache/solr/search/SolrDocumentFetcher.java
+++ b/solr/core/src/java/org/apache/solr/search/SolrDocumentFetcher.java
@@ -634,7 +634,7 @@ public class SolrDocumentFetcher {
case BINARY:
BinaryDocValues bdv = e.getBinaryDocValues(localId, leafReader,
readerOrd);
if (bdv != null) {
- return BytesRef.deepCopyOf(bdv.binaryValue()).bytes;
+ return e.schemaField.getType().toObject(e.schemaField,
bdv.binaryValue());
}
return null;
case SORTED:
diff --git
a/solr/core/src/test-files/solr/collection1/conf/schema-binaryfield.xml
b/solr/core/src/test-files/solr/collection1/conf/schema-binaryfield.xml
index 51c2b26a832..0046ecd8fd1 100644
--- a/solr/core/src/test-files/solr/collection1/conf/schema-binaryfield.xml
+++ b/solr/core/src/test-files/solr/collection1/conf/schema-binaryfield.xml
@@ -28,15 +28,23 @@
<schema name="test" version="1.7">
<fieldType name="binary" class="solr.BinaryField"/>
+ <fieldType name="binary_rev" class="solr.SwapBytesBinaryField" />
+ <fieldType name="binary_str" class="solr.StrBinaryField" />
+
<fieldType name="boolean" class="solr.BoolField" sortMissingLast="true"/>
<fieldType name="string" class="solr.StrField" sortMissingLast="true"/>
<field name="id" type="string" indexed="true" stored="true"
multiValued="false" required="true"/>
+
<field name="data" type="binary" stored="true"/>
<field name="data_dv" type="binary" stored="false" docValues="true" />
+ <field name="rev_data" type="binary_rev" stored="true"/>
+ <field name="rev_data_dv" type="binary_rev" stored="false" docValues="true"
/>
+
+ <field name="str_data" type="binary_str" stored="true"/>
+ <field name="str_data_dv" type="binary_str" stored="false" docValues="true"
/>
<uniqueKey>id</uniqueKey>
-
</schema>
diff --git
a/solr/core/src/test-files/solr/collection1/conf/schema-non-stored-docvalues.xml
b/solr/core/src/test-files/solr/collection1/conf/schema-non-stored-docvalues.xml
index aa9a3a9ea51..6a7987fa38d 100644
---
a/solr/core/src/test-files/solr/collection1/conf/schema-non-stored-docvalues.xml
+++
b/solr/core/src/test-files/solr/collection1/conf/schema-non-stored-docvalues.xml
@@ -28,6 +28,9 @@
<fieldType name="date" class="${solr.tests.DateFieldType}"
precisionStep="0"/>
<fieldType name="enumField" class="solr.EnumFieldType" docValues="true"
enumsConfig="enumsConfig.xml" enumName="severity"/>
+ <fieldType name="binary" class="solr.BinaryField" />
+ <fieldType name="binary_rev" class="solr.SwapBytesBinaryField" />
+ <fieldType name="binary_str" class="solr.StrBinaryField" />
<field name="id" type="string" indexed="true" stored="true"
multiValued="false" required="false"/>
@@ -65,6 +68,18 @@
<dynamicField name="*_ls_dvo" multiValued="true" type="long" indexed="true"
stored="false"/>
<dynamicField name="*_dts_dvo" multiValued="true" type="date" indexed="true"
stored="false"/>
+ <!-- binary fields (exclusively single valued) -->
+ <dynamicField name="*_bin" multiValued="false" type="binary" indexed="false"
stored="true" docValues="false" />
+ <dynamicField name="*_bin_dv" multiValued="false" type="binary"
indexed="false" stored="true" docValues="true" />
+ <dynamicField name="*_bin_dvo" multiValued="false" type="binary"
indexed="false" stored="false" docValues="true" />
+
+ <dynamicField name="*_rev_bin" multiValued="false" type="binary_rev"
indexed="false" stored="true" docValues="false" />
+ <dynamicField name="*_rev_bin_dv" multiValued="false" type="binary_rev"
indexed="false" stored="true" docValues="true" />
+ <dynamicField name="*_rev_bin_dvo" multiValued="false" type="binary_rev"
indexed="false" stored="false" docValues="true" />
+
+ <dynamicField name="*_str_bin" multiValued="false" type="binary_str"
indexed="false" stored="true" docValues="false" />
+ <dynamicField name="*_str_bin_dv" multiValued="false" type="binary_str"
indexed="false" stored="true" docValues="true" />
+ <dynamicField name="*_str_bin_dvo" multiValued="false" type="binary_str"
indexed="false" stored="false" docValues="true" />
<uniqueKey>id</uniqueKey>
diff --git a/solr/core/src/test/org/apache/solr/schema/StrBinaryField.java
b/solr/core/src/test/org/apache/solr/schema/StrBinaryField.java
new file mode 100644
index 00000000000..adc2b8cee20
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/schema/StrBinaryField.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.schema;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
+import java.util.Base64;
+import java.util.List;
+import org.apache.lucene.index.IndexableField;
+import org.apache.lucene.util.BytesRef;
+import org.apache.solr.response.TextResponseWriter;
+
+/**
+ * Custom binary field that is always Base64 stringified with external
clients, using a special
+ * prefix
+ */
+public final class StrBinaryField extends BinaryField {
+ public static final String PREFIX = "CUSTOMPRE_";
+
+ @Override
+ public void write(TextResponseWriter writer, String name, IndexableField f)
throws IOException {
+ writer.writeStr(name, toExternal(f), false);
+ }
+
+ @Override
+ public String toExternal(IndexableField f) {
+ return PREFIX + super.toExternal(f);
+ }
+
+ @Override
+ public Object toObject(SchemaField sf, BytesRef term) {
+ return PREFIX
+ + Base64.getEncoder()
+ .encodeToString(Arrays.copyOfRange(term.bytes, term.offset,
term.offset + term.length));
+ }
+
+ @Override
+ public List<IndexableField> createFields(SchemaField field, Object val) {
+ if (val instanceof String valStr) {
+ if (valStr.startsWith(PREFIX)) {
+ return super.createFields(field, valStr.substring(PREFIX.length()));
+ }
+ throw new RuntimeException(
+ field.getName() + " values must be strings PREFIXED with " + PREFIX
+ "; got: " + valStr);
+ }
+ throw new RuntimeException(
+ field.getName()
+ + " values must be STRINGS starting with "
+ + PREFIX
+ + "; got: "
+ + val.getClass());
+ }
+
+ @Override
+ public Object toNativeType(Object val) {
+ Object result = super.toNativeType(val);
+ if (result instanceof ByteBuffer buf) {
+ // Kludge because super doesn't give us access to it's method...
+ result =
+ new String(
+ Base64.getEncoder()
+ .encode(
+ ByteBuffer.wrap(
+ buf.array(),
+ buf.arrayOffset() + buf.position(),
+ buf.limit() - buf.position())
+ .array()),
+ StandardCharsets.ISO_8859_1);
+ }
+ return PREFIX + result.toString();
+ }
+}
diff --git
a/solr/core/src/test/org/apache/solr/schema/SwapBytesBinaryField.java
b/solr/core/src/test/org/apache/solr/schema/SwapBytesBinaryField.java
new file mode 100644
index 00000000000..55997dcfc21
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/schema/SwapBytesBinaryField.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.schema;
+
+import java.nio.ByteBuffer;
+import java.util.List;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.index.IndexableField;
+import org.apache.lucene.util.BytesRef;
+
+/**
+ * Custom binary field that internaly reverses the btes, but if all the I/O
layers of Solr work
+ * correctly, clients should never know
+ */
+public final class SwapBytesBinaryField extends BinaryField
+ implements FieldType.ExternalizeStoredValuesAsObjects {
+
+ public static byte[] copyAndReverse(final byte[] array, final int offset,
final int length) {
+ final byte[] result = new byte[length];
+ for (int i = 0; i < length; i++) {
+ result[i] = array[offset + length - 1 - i];
+ }
+ return result;
+ }
+
+ public static ByteBuffer copyAndReverse(final ByteBuffer in) {
+ return ByteBuffer.wrap(
+ copyAndReverse(in.array(), in.arrayOffset() + in.position(),
in.remaining()));
+ }
+
+ public static BytesRef copyAndReverse(final BytesRef in) {
+ return new BytesRef(copyAndReverse(in.bytes, in.offset, in.length));
+ }
+
+ @Override
+ public ByteBuffer toObject(IndexableField f) {
+ return copyAndReverse(super.toObject(f));
+ }
+
+ @Override
+ public Object toObject(SchemaField sf, BytesRef term) {
+ return copyAndReverse(term).bytes;
+ }
+
+ /**
+ * This is kludgy, but since BinaryField doesn't let us override
"getBytesRef(Object)", this is
+ * the simplest place for us to reverse things after super has generated
fields with binary values
+ * for us.
+ */
+ @Override
+ public List<IndexableField> createFields(SchemaField field, Object val) {
+ final List<IndexableField> results = super.createFields(field, val);
+ for (IndexableField indexable : results) {
+ if (indexable instanceof Field f) {
+ if (null != f.binaryValue()) {
+ f.setBytesValue(copyAndReverse(f.binaryValue()));
+ }
+ } else {
+ throw new RuntimeException(
+ "WTF: test is broken by unexpected type of IndexableField from
super");
+ }
+ }
+ return results;
+ }
+
+ @Override
+ public Object toNativeType(Object val) {
+ Object result = super.toNativeType(val);
+ if (result instanceof ByteBuffer originalBuf) {
+ result = copyAndReverse(originalBuf);
+ }
+ return result;
+ }
+}
diff --git a/solr/core/src/test/org/apache/solr/schema/TestBinaryField.java
b/solr/core/src/test/org/apache/solr/schema/TestBinaryField.java
index 0c830249ae0..759856dd8ae 100644
--- a/solr/core/src/test/org/apache/solr/schema/TestBinaryField.java
+++ b/solr/core/src/test/org/apache/solr/schema/TestBinaryField.java
@@ -20,7 +20,11 @@ import java.nio.ByteBuffer;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardCopyOption;
+import java.util.Arrays;
+import java.util.Base64;
+import java.util.HashMap;
import java.util.List;
+import java.util.Map;
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.SolrTestCaseJ4.SuppressSSL;
import org.apache.solr.client.solrj.SolrClient;
@@ -34,6 +38,7 @@ import org.apache.solr.util.SolrJettyTestRule;
import org.junit.BeforeClass;
import org.junit.ClassRule;
+/** Test "binary" based fields using SolrJ and javabn codec (with and w/o bean
mapping) */
@SuppressSSL(bugUrl = "https://issues.apache.org/jira/browse/SOLR-5776";)
public class TestBinaryField extends SolrTestCaseJ4 {
@@ -56,97 +61,116 @@ public class TestBinaryField extends SolrTestCaseJ4 {
solrTestRule.startSolr(homeDir);
}
+ /**
+ * @see TestUseDocValuesAsStored#testBinary
+ */
public void testSimple() throws Exception {
try (SolrClient client = solrTestRule.getSolrClient()) {
byte[] buf = new byte[10];
for (int i = 0; i < 10; i++) {
buf[i] = (byte) i;
}
+
+ final Map<String, byte[]> expected = new HashMap<>();
+
SolrInputDocument doc = null;
doc = new SolrInputDocument();
- doc.addField("id", 1);
+ doc.addField("id", "1");
+ expected.put("1", Arrays.copyOfRange(buf, 2, (2 + 5)));
doc.addField("data", ByteBuffer.wrap(buf, 2, 5));
doc.addField("data_dv", ByteBuffer.wrap(buf, 2, 5));
+ doc.addField("rev_data", ByteBuffer.wrap(buf, 2, 5));
+ doc.addField("rev_data_dv", ByteBuffer.wrap(buf, 2, 5));
+ doc.addField("str_data", prefix_base64(Arrays.copyOfRange(buf, 2, (2 +
5))));
+ doc.addField("str_data_dv", prefix_base64(Arrays.copyOfRange(buf, 2, (2
+ 5))));
client.add(doc);
doc = new SolrInputDocument();
- doc.addField("id", 2);
+ doc.addField("id", "2");
+ expected.put("2", Arrays.copyOfRange(buf, 4, (4 + 3)));
doc.addField("data", ByteBuffer.wrap(buf, 4, 3));
doc.addField("data_dv", ByteBuffer.wrap(buf, 4, 3));
+ doc.addField("rev_data", ByteBuffer.wrap(buf, 4, 3));
+ doc.addField("rev_data_dv", ByteBuffer.wrap(buf, 4, 3));
+ doc.addField("str_data", prefix_base64(Arrays.copyOfRange(buf, 4, (4 +
3))));
+ doc.addField("str_data_dv", prefix_base64(Arrays.copyOfRange(buf, 4, (4
+ 3))));
client.add(doc);
doc = new SolrInputDocument();
- doc.addField("id", 3);
+ doc.addField("id", "3");
+ expected.put("3", Arrays.copyOf(buf, buf.length));
doc.addField("data", buf);
doc.addField("data_dv", buf);
+ doc.addField("rev_data", buf);
+ doc.addField("rev_data_dv", buf);
+ doc.addField("str_data", prefix_base64(buf));
+ doc.addField("str_data_dv", prefix_base64(buf));
client.add(doc);
client.commit();
- QueryResponse resp = client.query(new SolrQuery("*:*").setFields("id",
"data", "data_dv"));
+ QueryResponse resp =
+ client.query(
+ new SolrQuery("*:*")
+ .setFields(
+ "id",
+ "data",
+ "data_dv",
+ "rev_data",
+ "rev_data_dv",
+ "str_data",
+ "str_data_dv"));
SolrDocumentList res = resp.getResults();
List<Bean> beans = resp.getBeans(Bean.class);
assertEquals(3, res.size());
assertEquals(3, beans.size());
+
for (SolrDocument d : res) {
- int id = Integer.parseInt(d.getFieldValue("id").toString());
- for (String field : new String[] {"data", "data_dv"}) {
- byte[] data = (byte[]) d.getFieldValue(field);
- if (id == 1) {
- assertEquals(5, data.length);
- for (int i = 0; i < data.length; i++) {
- byte b = data[i];
- assertEquals((byte) (i + 2), b);
- }
-
- } else if (id == 2) {
- assertEquals(3, data.length);
- for (int i = 0; i < data.length; i++) {
- byte b = data[i];
- assertEquals((byte) (i + 4), b);
- }
-
- } else if (id == 3) {
- assertEquals(10, data.length);
- for (int i = 0; i < data.length; i++) {
- byte b = data[i];
- assertEquals((byte) i, b);
- }
- }
- }
+ final String id = d.getFieldValue("id").toString();
+ assertTrue("Unexpected id: " + id, expected.containsKey(id));
+ final byte[] expected_bytes = expected.get(id);
+ final String expected_string = prefix_base64(expected_bytes);
+
+ assertArrayEquals(expected_bytes, (byte[]) d.getFieldValue("data"));
+ assertArrayEquals(expected_bytes, (byte[]) d.getFieldValue("data_dv"));
+
+ assertArrayEquals(expected_bytes, (byte[])
d.getFieldValue("rev_data"));
+ assertArrayEquals(expected_bytes, (byte[])
d.getFieldValue("rev_data_dv"));
+
+ assertEquals(expected_string, d.getFieldValue("str_data"));
+ assertEquals(expected_string, d.getFieldValue("str_data_dv"));
}
for (Bean d : beans) {
- int id = Integer.parseInt(d.id);
- for (byte[] data : new byte[][] {d.data, d.data_dv}) {
- if (id == 1) {
- assertEquals(5, data.length);
- for (int i = 0; i < data.length; i++) {
- byte b = data[i];
- assertEquals((byte) (i + 2), b);
- }
-
- } else if (id == 2) {
- assertEquals(3, data.length);
- for (int i = 0; i < data.length; i++) {
- byte b = data[i];
- assertEquals((byte) (i + 4), b);
- }
-
- } else if (id == 3) {
- assertEquals(10, data.length);
- for (int i = 0; i < data.length; i++) {
- byte b = data[i];
- assertEquals((byte) i, b);
- }
- }
- }
+ assertTrue("Unexpected id: " + d.id, expected.containsKey(d.id));
+ final byte[] expected_bytes = expected.get(d.id);
+ final String expected_string = prefix_base64(expected_bytes);
+
+ assertArrayEquals(expected_bytes, d.data);
+ assertArrayEquals(expected_bytes, d.data_dv);
+
+ assertArrayEquals(expected_bytes, d.rev_data);
+ assertArrayEquals(expected_bytes, d.rev_data_dv);
+
+ assertEquals(expected_string, d.str_data);
+ assertEquals(expected_string, d.str_data_dv);
}
}
}
+ /**
+ * @see StrBinaryField
+ */
+ public static String prefix_base64(final byte[] val) {
+ return StrBinaryField.PREFIX + Base64.getEncoder().encodeToString(val);
+ }
+
public static class Bean {
@Field String id;
@Field byte[] data;
@Field byte[] data_dv;
+ @Field byte[] rev_data;
+ @Field byte[] rev_data_dv;
+ @Field String str_data;
+ @Field String str_data_dv;
}
}
diff --git
a/solr/core/src/test/org/apache/solr/schema/TestUseDocValuesAsStored.java
b/solr/core/src/test/org/apache/solr/schema/TestUseDocValuesAsStored.java
index 59c40f16edc..296684404de 100644
--- a/solr/core/src/test/org/apache/solr/schema/TestUseDocValuesAsStored.java
+++ b/solr/core/src/test/org/apache/solr/schema/TestUseDocValuesAsStored.java
@@ -16,7 +16,9 @@
*/
package org.apache.solr.schema;
+import java.io.IOException;
import java.io.InputStream;
+import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
@@ -25,6 +27,7 @@ import java.time.LocalDateTime;
import java.time.Month;
import java.time.ZoneOffset;
import java.util.Arrays;
+import java.util.Base64;
import java.util.HashSet;
import java.util.Set;
import java.util.regex.Pattern;
@@ -34,11 +37,19 @@ import javax.xml.xpath.XPath;
import javax.xml.xpath.XPathConstants;
import javax.xml.xpath.XPathFactory;
import org.apache.commons.io.file.PathUtils;
+import org.apache.lucene.index.BinaryDocValues;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexableField;
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.index.StoredFields;
import org.apache.lucene.tests.mockfile.FilterPath;
import org.apache.lucene.tests.util.TestUtil;
+import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.IOUtils;
import org.apache.solr.common.util.DOMUtil;
import org.apache.solr.core.AbstractBadConfigTestBase;
+import org.apache.solr.core.SolrCore;
+import org.apache.solr.search.SolrIndexSearcher;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
@@ -46,7 +57,11 @@ import org.w3c.dom.Document;
import org.w3c.dom.NodeList;
import org.xml.sax.InputSource;
-/** Tests the useDocValuesAsStored functionality. */
+/**
+ * Tests the useDocValuesAsStored functionality.
+ *
+ * <p>This test uses XML/json (text writer) based responses.
+ */
public class TestUseDocValuesAsStored extends AbstractBadConfigTestBase {
private int id = 1;
@@ -241,6 +256,96 @@ public class TestUseDocValuesAsStored extends
AbstractBadConfigTestBase {
}
}
+ /**
+ * @see TestBinaryField
+ * @see #testInternallyReversedBytesBinary
+ * @see #testExternallyPrefixedStringifiedBinary
+ */
+ public void testBinary() throws Exception {
+
+ // When indexing: BinaryField can parse base64 encoded string values
+ // In XML/json output: response values will also be base64 encoded
+ final String data = Base64.getEncoder().encodeToString(new byte[] {0, 0,
1, 2, 3});
+
+ doTest("binary stored only", "foo_bin", "str", data);
+ doTest("binary stored+DV", "foo_bin_dv", "str", data);
+ doTest("binary DV only", "foo_bin_dvo", "str", data);
+
+ // sanity check our low level internal values
+ final BytesRef expectedInternalValue = new BytesRef(new byte[] {0, 0, 1,
2, 3});
+ assertAllInternalBinaryValuesMatch("foo_bin", expectedInternalValue);
+ assertAllInternalBinaryValuesMatch("foo_bin_dv", expectedInternalValue);
+ assertAllInternalBinaryValuesMatch("foo_bin_dvo", expectedInternalValue);
+ }
+
+ /**
+ * @see TestBinaryField
+ * @see #testBinary
+ * @see #testInternallyReversedBytesBinary
+ */
+ public void testExternallyPrefixedStringifiedBinary() throws Exception {
+
+ // This field type *enforces* base64 encoded binary data, with a custom
prefix
+ final String data =
+ StrBinaryField.PREFIX + Base64.getEncoder().encodeToString(new byte[]
{0, 0, 1, 2, 3});
+
+ doTest("(str) binary stored only", "foo_str_bin", "str", data);
+ doTest("(str) binary stored+DV", "foo_str_bin_dv", "str", data);
+ doTest("(str) binary DV only", "foo_str_bin_dvo", "str", data);
+
+ // sanity check our low level internal values
+ final BytesRef expectedInternalValue = new BytesRef(new byte[] {0, 0, 1,
2, 3});
+ assertAllInternalBinaryValuesMatch("foo_str_bin", expectedInternalValue);
+ assertAllInternalBinaryValuesMatch("foo_str_bin_dv",
expectedInternalValue);
+ assertAllInternalBinaryValuesMatch("foo_str_bin_dvo",
expectedInternalValue);
+ }
+
+ /**
+ * @see TestBinaryField
+ * @see #testBinary
+ * @see #testExternallyPrefixedStringifiedBinary
+ */
+ public void testInternallyReversedBytesBinary() throws Exception {
+
+ // When indexing: BinaryField can parse base64 encoded string values
+ // In XML/json output: response values will also be base64 encoded
+ final String data = Base64.getEncoder().encodeToString(new byte[] {0, 0,
1, 2, 3});
+
+ doTest("(rev) binary stored only", "foo_rev_bin", "str", data);
+ doTest("(rev) binary stored+DV", "foo_rev_bin_dv", "str", data);
+ doTest("(rev) binary DV only", "foo_rev_bin_dvo", "str", data);
+
+ // sanity check our low level internal values
+ final BytesRef expectedInternalValue = new BytesRef(new byte[] {3, 2, 1,
0, 0});
+ assertAllInternalBinaryValuesMatch("foo_rev_bin", expectedInternalValue);
+ assertAllInternalBinaryValuesMatch("foo_rev_bin_dv",
expectedInternalValue);
+ assertAllInternalBinaryValuesMatch("foo_rev_bin_dvo",
expectedInternalValue);
+ }
+
+ public void testSanityCheckOfSwapBytesBinaryField() throws Exception {
+ final byte[] expected = new byte[] {1, 2, 3, 4, 5};
+
+ assertArrayEquals(
+ expected, SwapBytesBinaryField.copyAndReverse(new byte[] {5, 4, 3, 2,
1}, 0, 5));
+ assertArrayEquals(
+ expected, SwapBytesBinaryField.copyAndReverse(new byte[] {9, 5, 4, 3,
2, 1, 0}, 1, 5));
+
+ assertEquals(
+ ByteBuffer.wrap(expected),
+ SwapBytesBinaryField.copyAndReverse(ByteBuffer.wrap(new byte[] {5, 4,
3, 2, 1})));
+ assertEquals(
+ ByteBuffer.wrap(expected),
+ SwapBytesBinaryField.copyAndReverse(
+ ByteBuffer.wrap(new byte[] {9, 5, 4, 3, 2, 1, 0}, 1, 5)));
+
+ assertEquals(
+ new BytesRef(expected),
+ SwapBytesBinaryField.copyAndReverse(new BytesRef(new byte[] {5, 4, 3,
2, 1})));
+ assertEquals(
+ new BytesRef(expected),
+ SwapBytesBinaryField.copyAndReverse(new BytesRef(new byte[] {9, 5, 4,
3, 2, 1, 0}, 1, 5)));
+ }
+
private String plural(int arity) {
return arity > 1 ? "s" : "";
}
@@ -540,4 +645,31 @@ public class TestUseDocValuesAsStored extends
AbstractBadConfigTestBase {
"/response/docs/[0]/test_mvt_dvu_st_str/[1]==aaaa",
"/response/docs/[0]/test_mvt_dvu_st_str/[2]==bbbb");
}
+
+ /**
+ * A fairly niche helper method that asserts all binary doc values and/or
stored values in the
+ * specified field (regardless of doc id) all match an explicitly expected
value.
+ */
+ private final void assertAllInternalBinaryValuesMatch(
+ final String fieldName, final BytesRef expected) throws IOException {
+ try (SolrCore core = h.getCoreInc()) {
+ final IndexReader top =
core.withSearcher(SolrIndexSearcher::getIndexReader);
+ final StoredFields stored = top.storedFields();
+ for (int id = 0; id < top.maxDoc(); id++) {
+ final IndexableField storedValue =
stored.document(id).getField(fieldName);
+ // we might be called on a non-stored field, but if it is stored it
better be binary
+ if (null != storedValue) {
+ assertEquals(expected, storedValue.binaryValue());
+ }
+ }
+ for (LeafReaderContext context : top.leaves()) {
+ // Note: explicitly avoid DocValues.getBinary(...) to bypass type
check.
+ // For our purposes, we're fine with NONE if the field type isn't
using docValues at all
+ final BinaryDocValues dv =
context.reader().getBinaryDocValues(fieldName);
+ while (null != dv && dv.nextDoc() < BinaryDocValues.NO_MORE_DOCS) {
+ assertEquals(expected, dv.binaryValue());
+ }
+ }
+ }
+ }
}
