This is an automated email from the ASF dual-hosted git repository. dsmiley pushed a commit to branch branch_9x in repository https://gitbox.apache.org/repos/asf/solr.git
commit 1899409a4730e9ffbe95c0bafeb2b87ea01c9bb2 Author: David Smiley <[email protected]> AuthorDate: Tue Oct 20 21:49:08 2026 +0100 SOLR-9957: /export supports document transformers in fl (#5113) Albeit some might not work. Co-authored-by: Claude Opus 5.5 (1M context) <[email protected]> (cherry picked from commit bd2b2d5a02584893c5b5fe16c1a7eee89c2bb6c9) (cherry picked from commit cf49460e11e83e938cef0c562bda83d0f18b1abc) Adapted to Java 11 and branch_9x's DocTransformer API; tests use qt=/export. --- .../unreleased/SOLR-9957-exportTransformers.yml | 10 ++ .../apache/solr/handler/export/ExportWriter.java | 152 +++++++++++++++++---- .../solr/handler/export/ExportWriterStream.java | 16 +-- .../solr/handler/export/TestExportWriter.java | 77 +++++++++++ .../query-guide/pages/exporting-result-sets.adoc | 4 + 5 files changed, 215 insertions(+), 44 deletions(-) diff --git a/changelog/unreleased/SOLR-9957-exportTransformers.yml b/changelog/unreleased/SOLR-9957-exportTransformers.yml new file mode 100644 index 00000000000..aa37e8f4183 --- /dev/null +++ b/changelog/unreleased/SOLR-9957-exportTransformers.yml @@ -0,0 +1,10 @@ +title: > + The /export handler now supports document transformers (e.g. `[docid]`) and field aliases in `fl`, + and their output is visible to its local streaming expression (`expr`). An aliased field such as + `fl=x:myfield` is now returned as `x`; previously it was silently returned as `myfield`. +type: added +authors: + - name: David Smiley +links: + - name: SOLR-9957 + url: https://issues.apache.org/jira/browse/SOLR-9957 diff --git a/solr/core/src/java/org/apache/solr/handler/export/ExportWriter.java b/solr/core/src/java/org/apache/solr/handler/export/ExportWriter.java index 9307f167b48..c06ae23d0fa 100644 --- a/solr/core/src/java/org/apache/solr/handler/export/ExportWriter.java +++ b/solr/core/src/java/org/apache/solr/handler/export/ExportWriter.java @@ -28,6 +28,7 @@ import java.io.PrintWriter; import java.lang.invoke.MethodHandles; import java.nio.charset.StandardCharsets; import java.util.ArrayList; +import java.util.HashSet; import java.util.LinkedHashMap; import java.util.LinkedHashSet; import java.util.List; @@ -38,6 +39,7 @@ import org.apache.lucene.index.LeafReader; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.SortedDocValues; import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.search.Query; import org.apache.lucene.search.Sort; import org.apache.lucene.search.SortField; import org.apache.lucene.util.BitSetIterator; @@ -54,10 +56,12 @@ import org.apache.solr.common.IteratorWriter; import org.apache.solr.common.MapWriter; import org.apache.solr.common.MapWriter.EntryWriter; import org.apache.solr.common.PushWriter; +import org.apache.solr.common.SolrDocument; import org.apache.solr.common.params.CommonParams; import org.apache.solr.common.params.SolrParams; import org.apache.solr.common.params.StreamParams; import org.apache.solr.common.util.JavaBinCodec; +import org.apache.solr.common.util.SimpleOrderedMap; import org.apache.solr.core.SolrCore; import org.apache.solr.metrics.SolrMetricsContext; import org.apache.solr.request.SolrQueryRequest; @@ -66,7 +70,9 @@ import org.apache.solr.response.BinaryResponseWriter; import org.apache.solr.response.JSONResponseWriter; import org.apache.solr.response.JacksonJsonWriter; import org.apache.solr.response.QueryResponseWriter; +import org.apache.solr.response.ResultContext; import org.apache.solr.response.SolrQueryResponse; +import org.apache.solr.response.transform.DocTransformer; import org.apache.solr.schema.BoolField; import org.apache.solr.schema.DateValueFieldType; import org.apache.solr.schema.DoubleValueFieldType; @@ -78,7 +84,9 @@ import org.apache.solr.schema.LongValueFieldType; import org.apache.solr.schema.SchemaField; import org.apache.solr.schema.SortableTextField; import org.apache.solr.schema.StrField; +import org.apache.solr.search.DocList; import org.apache.solr.search.DocValuesIteratorCache; +import org.apache.solr.search.ReturnFields; import org.apache.solr.search.SolrIndexSearcher; import org.apache.solr.search.SolrReturnFields; import org.apache.solr.search.SortSpec; @@ -126,7 +134,9 @@ public class ExportWriter implements SolrCore.RawWriter, Closeable { private int priorityQueueSize; StreamExpression streamExpression; StreamContext streamContext; + SolrReturnFields returnFields; List<FieldWriter> fieldWriters; + DocTransformer transformer; int totalHits = 0; FixedBitSet[] sets = null; PushWriter writer; @@ -270,35 +280,38 @@ public class ExportWriter implements SolrCore.RawWriter, Closeable { } } SolrParams params = req.getParams(); - String fl = params.get("fl"); - String[] fields = null; - - if (fl == null) { + if (params.get(CommonParams.FL) == null) { writeException( (new IOException(new SyntaxError("export field list (fl) must be specified."))), writer, true); return; - } else { - fields = fl.split(","); - - for (int i = 0; i < fields.length; i++) { + } - fields[i] = fields[i].trim(); + try { + returnFields = new SolrReturnFields(req); + } catch (Exception e) { + writeException(e, writer, true); + return; + } - if (fields[i].equals("score")) { - writeException( - (new IOException(new SyntaxError("Scoring is not currently supported with xsort."))), - writer, - true); - return; - } - } + if (returnFields.wantsScore()) { + writeException( + (new IOException(new SyntaxError("Scoring is not currently supported with xsort."))), + writer, + true); + return; } try { - fieldWriters = getFieldWriters(fields, req); + fieldWriters = getFieldWriters(returnFields, req); + transformer = returnFields.getTransformer(); + if (transformer != null) { + transformer.setContext( + new ExportResultContext( + returnFields, searcher, info.getResponseBuilder().getQuery(), req)); + } } catch (Exception e) { writeException(e, writer, true); return; @@ -485,33 +498,73 @@ public class ExportWriter implements SolrCore.RawWriter, Closeable { throws IOException { int ord = sortDoc.ord; LeafReaderContext context = leaves.get(ord); + if (transformer == null) { + for (FieldWriter fieldWriter : writers) { + fieldWriter.write(sortDoc, context, ew); + } + return; + } + + SolrDocument doc = new SolrDocument(); + EntryWriter docWriter = + new EntryWriter() { + @Override + public EntryWriter put(CharSequence k, Object v) { + doc.setField(k.toString(), materialize(v)); + return this; + } + }; for (FieldWriter fieldWriter : writers) { - fieldWriter.write(sortDoc, context, ew); + fieldWriter.write(sortDoc, context, docWriter); + } + transformer.transform(doc, sortDoc.docBase + sortDoc.docId); + for (Map.Entry<String, Object> entry : doc) { + if (returnFields.wantsField(entry.getKey())) { + ew.put(entry.getKey(), entry.getValue()); + } } } - public List<FieldWriter> getFieldWriters(String[] fields, SolrQueryRequest req) + /** Converts lazily written values to plain Java objects that can outlive the write call. */ + static Object materialize(Object v) { + if (v instanceof IteratorWriter) { + List<Object> lst = new ArrayList<>(); + ((IteratorWriter) v).toList(lst); + return lst; + } else if (v instanceof MapWriter) { + return new SimpleOrderedMap<>((MapWriter) v); + } + return v; + } + + List<FieldWriter> getFieldWriters(SolrReturnFields solrReturnFields, SolrQueryRequest req) throws IOException { DocValuesIteratorCache dvIterCache = new DocValuesIteratorCache(req.getSearcher(), false); - SolrReturnFields solrReturnFields = new SolrReturnFields(fields, req); boolean includeStoredFields = req.getParams().getBool(INCLUDE_STORED_FIELDS_PARAM, false); + // Also fetch fields only a transformer reads, e.g. the f= of [geo]; wantsField excludes them + Set<String> fieldsToFetch = + solrReturnFields.getLuceneFieldNames() != null + ? solrReturnFields.getLuceneFieldNames() + : Set.of(); + // Non-exportable fields named here error; ones matched only by a glob are skipped. + // Includes fieldsToFetch, since requested names hold an alias, not its source field. + Set<String> explicitlyNamedFields = new HashSet<>(fieldsToFetch); + if (solrReturnFields.getRequestedFieldNames() != null) { + explicitlyNamedFields.addAll(solrReturnFields.getRequestedFieldNames()); + } + List<FieldWriter> writers = new ArrayList<>(); Set<SchemaField> docValueFields = new LinkedHashSet<>(); Map<String, SchemaField> storedFields = new LinkedHashMap<>(); for (String field : req.getSearcher().getFieldNames()) { - if (!solrReturnFields.wantsField(field)) { + if (!solrReturnFields.wantsField(field) && !fieldsToFetch.contains(field)) { continue; } SchemaField schemaField = req.getSchema().getField(field); FieldType fieldType = schemaField.getType(); - Set<String> requestFieldNames = - solrReturnFields.getRequestedFieldNames() == null - ? Set.of() - : solrReturnFields.getRequestedFieldNames(); - if (canUseDocValues(schemaField, fieldType)) { // Prefer DocValues when available docValueFields.add(schemaField); @@ -519,7 +572,7 @@ public class ExportWriter implements SolrCore.RawWriter, Closeable { // Field is stored-only (no usable DocValues) if (includeStoredFields) { storedFields.put(field, schemaField); - } else if (requestFieldNames.contains(field)) { + } else if (explicitlyNamedFields.contains(field)) { // Explicitly requested field without DocValues and includeStoredFields=false throw new IOException( schemaField @@ -527,7 +580,7 @@ public class ExportWriter implements SolrCore.RawWriter, Closeable { + "Try setting includeStoredFields=true to retrieve this field from stored values."); } // Else: glob matched stored-only field without includeStoredFields - silently skip - } else if (requestFieldNames.contains(field)) { + } else if (explicitlyNamedFields.contains(field)) { // Explicitly requested field that has neither DocValues nor stored if (fieldType instanceof SortableTextField && !schemaField.useDocValuesAsStored()) { throw new IOException( @@ -612,6 +665,47 @@ public class ExportWriter implements SolrCore.RawWriter, Closeable { return writers; } + /** Context for transformers in {@code fl}; there is no {@link DocList} when exporting. */ + private static final class ExportResultContext extends ResultContext { + private final ReturnFields returnFields; + private final SolrIndexSearcher searcher; + private final Query query; + private final SolrQueryRequest req; + + ExportResultContext( + ReturnFields returnFields, SolrIndexSearcher searcher, Query query, SolrQueryRequest req) { + this.returnFields = returnFields; + this.searcher = searcher; + this.query = query; + this.req = req; + } + + @Override + public DocList getDocList() { + return null; + } + + @Override + public ReturnFields getReturnFields() { + return returnFields; + } + + @Override + public SolrIndexSearcher getSearcher() { + return searcher; + } + + @Override + public Query getQuery() { + return query; + } + + @Override + public SolrQueryRequest getRequest() { + return req; + } + } + private static boolean canUseDocValues(SchemaField schemaField, FieldType fieldType) { return schemaField.hasDocValues() // Special handling for SortableTextField: unlike other field types, it requires diff --git a/solr/core/src/java/org/apache/solr/handler/export/ExportWriterStream.java b/solr/core/src/java/org/apache/solr/handler/export/ExportWriterStream.java index 8700064bba7..3c21cb8202c 100644 --- a/solr/core/src/java/org/apache/solr/handler/export/ExportWriterStream.java +++ b/solr/core/src/java/org/apache/solr/handler/export/ExportWriterStream.java @@ -18,10 +18,7 @@ package org.apache.solr.handler.export; import java.io.IOException; import java.lang.invoke.MethodHandles; -import java.util.ArrayList; -import java.util.HashMap; import java.util.List; -import java.util.Map; import java.util.concurrent.BrokenBarrierException; import java.util.concurrent.TimeoutException; import org.apache.solr.client.solrj.io.Tuple; @@ -37,8 +34,6 @@ import org.apache.solr.client.solrj.io.stream.expr.StreamExplanation; import org.apache.solr.client.solrj.io.stream.expr.StreamExpression; import org.apache.solr.client.solrj.io.stream.expr.StreamExpressionParameter; import org.apache.solr.client.solrj.io.stream.expr.StreamFactory; -import org.apache.solr.common.IteratorWriter; -import org.apache.solr.common.MapWriter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -66,16 +61,7 @@ public class ExportWriterStream extends TupleStream implements Expressible { @Override public EntryWriter put(CharSequence k, Object v) throws IOException { - if (v instanceof IteratorWriter) { - List<Object> lst = new ArrayList<>(); - ((IteratorWriter) v).toList(lst); - v = lst; - } else if (v instanceof MapWriter) { - Map<String, Object> map = new HashMap<>(); - ((MapWriter) v).toMap(map); - v = map; - } - tuple.put(k.toString(), v); + tuple.put(k.toString(), ExportWriter.materialize(v)); return this; } } diff --git a/solr/core/src/test/org/apache/solr/handler/export/TestExportWriter.java b/solr/core/src/test/org/apache/solr/handler/export/TestExportWriter.java index 8829d7e5c60..8273a997189 100644 --- a/solr/core/src/test/org/apache/solr/handler/export/TestExportWriter.java +++ b/solr/core/src/test/org/apache/solr/handler/export/TestExportWriter.java @@ -1333,6 +1333,83 @@ public class TestExportWriter extends SolrTestCaseJ4 { jq.contains("\"sortabledv\"")); } + @Test + @SuppressWarnings("unchecked") + public void testDocTransformers() throws Exception { + assertU(delQ("*:*")); + assertU(adoc("id", "1", "intdv", "10", "intdv_m", "1", "intdv_m", "2")); + assertU(adoc("id", "2", "intdv", "20")); + assertU(commit()); + assertU(adoc("id", "3", "intdv", "30")); + assertU(commit()); + + Map<String, Object> selectDocIds = new HashMap<>(); + String selectRsp = h.query(req("q", "*:*", "fl", "id,[docid]", "wt", "json")); + for (Map<String, Object> doc : + (List<Map<String, Object>>) + Utils.getObjectByPath( + mapper.readValue(selectRsp, Map.class), false, "/response/docs")) { + selectDocIds.put((String) doc.get("id"), doc.get("[docid]")); + } + assertEquals(3, selectDocIds.size()); + + // [docid] matches /select's, and fields are materialized (incl. multiValued) + List<Map<String, Object>> docs = + (List<Map<String, Object>>) + queryJsonReturnDocs( + req("qt", "/export", "q", "*:*", "fl", "id,intdv_m,[docid]", "sort", "id asc")); + assertEquals(3, docs.size()); + for (Map<String, Object> doc : docs) { + assertEquals(doc.toString(), selectDocIds.get(doc.get("id")), doc.get("[docid]")); + } + assertEquals(List.of(1, 2), docs.get(0).get("intdv_m")); + + // alias renames; transformer local params may contain commas + docs = + (List<Map<String, Object>>) + queryJsonReturnDocs( + req( + "qt", + "/export", + "q", + "*:*", + "fl", + "id,x:intdv,v:[value v='a,b']", + "sort", + "id asc")); + assertEquals(3, docs.size()); + assertEquals(Map.of("id", "1", "x", 10, "v", "a,b"), docs.get(0)); + + // transformed fields are visible to streaming expressions + docs = + (List<Map<String, Object>>) + queryJsonReturnDocs( + req( + "qt", + "/export", + "q", + "*:*", + "fl", + "id,[docid]", + "sort", + "id asc", + "expr", + "top(n=2,input(),sort=\"id desc\")")); + assertEquals(2, docs.size()); + assertEquals("3", docs.get(0).get("id")); + assertEquals(selectDocIds.get("3"), docs.get(0).get("[docid]")); + + // score is still rejected + docs = + (List<Map<String, Object>>) + queryJsonReturnDocs( + req("qt", "/export", "q", "*:*", "fl", "id,score", "sort", "id asc")); + assertEquals(1, docs.size()); + assertTrue( + docs.get(0).toString(), + docs.get(0).get(StreamParams.EXCEPTION).toString().contains("Scoring")); + } + @SuppressWarnings("rawtypes") private void validateSort(int numDocs) throws Exception { // 10 fields diff --git a/solr/solr-ref-guide/modules/query-guide/pages/exporting-result-sets.adoc b/solr/solr-ref-guide/modules/query-guide/pages/exporting-result-sets.adoc index fc6f4d6a7ef..c5ebf6dddfb 100644 --- a/solr/solr-ref-guide/modules/query-guide/pages/exporting-result-sets.adoc +++ b/solr/solr-ref-guide/modules/query-guide/pages/exporting-result-sets.adoc @@ -85,6 +85,10 @@ Note that sort fields still require docValues, regardless of this setting. Wildcard patterns can be used for the field list (e.g. `fl=*_i`) and will be expanded to the list of fields that match the pattern and are able to be exported, see <<Field Requirements>>. +Field aliases (e.g. `fl=id,price:price_d`) and xref:document-transformers.adoc[document transformers] (e.g. `fl=id,[docid]`) are supported, and their output is also visible to a local streaming expression. +Transformers are applied one document at a time, and those that depend on scores or on a list of matching documents won't work. +Some transformers, like `[subquery]` and `[child]`, produce output that might not work well, or at all, with `/export` and streaming expressions. + Returning scores is not supported at this time. === Specifying the Local Streaming Expression
