This is an automated email from the ASF dual-hosted git repository. jsweeney pushed a commit to branch SOLR-17022-glob-patterns-export-stream-handlers in repository https://gitbox.apache.org/repos/asf/solr.git
commit b210b1e6e0c3775e0628a72703a7a34da3a3b0c7 Author: Justin Sweeney <[email protected]> AuthorDate: Tue Oct 10 13:48:50 2023 -0400 Allowing for glob patterns for fields in ExportWriter --- .../apache/solr/handler/export/ExportWriter.java | 77 +++++++++++++++++++--- .../solr/handler/export/TestExportWriter.java | 37 +++++++++++ 2 files changed, 104 insertions(+), 10 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/handler/export/ExportWriter.java b/solr/core/src/java/org/apache/solr/handler/export/ExportWriter.java index 51ba5551b69..14ad566f444 100644 --- a/solr/core/src/java/org/apache/solr/handler/export/ExportWriter.java +++ b/solr/core/src/java/org/apache/solr/handler/export/ExportWriter.java @@ -27,9 +27,14 @@ import java.io.OutputStreamWriter; import java.io.PrintWriter; import java.lang.invoke.MethodHandles; import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.TreeSet; +import org.apache.commons.io.FilenameUtils; +import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.LeafReader; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.SortedDocValues; @@ -487,19 +492,14 @@ public class ExportWriter implements SolrCore.RawWriter, Closeable { public FieldWriter[] getFieldWriters(String[] fields, SolrIndexSearcher searcher) throws IOException { - IndexSchema schema = searcher.getSchema(); - FieldWriter[] writers = new FieldWriter[fields.length]; DocValuesIteratorCache dvIterCache = new DocValuesIteratorCache(searcher, false); - for (int i = 0; i < fields.length; i++) { - String field = fields[i]; - SchemaField schemaField = null; - try { - schemaField = schema.getField(field); - } catch (Exception e) { - throw new IOException(e); - } + List<SchemaField> expandedFields = expandFieldList(fields, searcher); + FieldWriter[] writers = new FieldWriter[expandedFields.size()]; + for (int i = 0; i < expandedFields.size(); i++) { + SchemaField schemaField = expandedFields.get(i); + String field = schemaField.getName(); if (!schemaField.hasDocValues()) { throw new IOException(schemaField + " must have DocValues to use this feature."); } @@ -844,4 +844,61 @@ public class ExportWriter implements SolrCore.RawWriter, Closeable { return "Early Client Disconnect"; } } + + /** + * Creates a complete field list using the provided field list by expanding any glob patterns into + * field names + * + * @param fields the original set of fields provided + * @param searcher an index searcher to access schema info + * @return a complete list of fields included any fields matching glob patterns + * @throws IOException if a provided field does not exist or cannot be retrieved from the schema + * info + */ + private List<SchemaField> expandFieldList(String[] fields, SolrIndexSearcher searcher) + throws IOException { + List<SchemaField> expandedFields = new ArrayList<>(fields.length); + Set<String> fieldsProcessed = new HashSet<>(); + for (String field : fields) { + try { + if (field.contains("*")) { + getGlobFields(field, searcher, fieldsProcessed, expandedFields); + } else { + if (fieldsProcessed.add(field)) { + expandedFields.add(searcher.getSchema().getField(field)); + } + } + } catch (Exception e) { + throw new IOException(e); + } + } + + return expandedFields; + } + + /** + * Create a list of schema fields that match a given glob pattern + * + * @param fieldPattern the glob pattern to match + * @param searcher an index search to access schema info + * @param fieldsProcessed the set of field names already processed to avoid duplicating + * @param expandedFields the list of fields to add expanded field names into + */ + private void getGlobFields( + String fieldPattern, + SolrIndexSearcher searcher, + Set<String> fieldsProcessed, + List<SchemaField> expandedFields) { + for (FieldInfo fi : searcher.getFieldInfos()) { + if (FilenameUtils.wildcardMatch(fi.getName(), fieldPattern)) { + SchemaField schemaField = searcher.getSchema().getField(fi.getName()); + if (fieldsProcessed.add(fi.getName()) + && schemaField.hasDocValues() + && (!(schemaField.getType() instanceof SortableTextField) + || schemaField.useDocValuesAsStored())) { + expandedFields.add(schemaField); + } + } + } + } } diff --git a/solr/core/src/test/org/apache/solr/handler/export/TestExportWriter.java b/solr/core/src/test/org/apache/solr/handler/export/TestExportWriter.java index 8337609faf9..e37f26efc94 100644 --- a/solr/core/src/test/org/apache/solr/handler/export/TestExportWriter.java +++ b/solr/core/src/test/org/apache/solr/handler/export/TestExportWriter.java @@ -1298,6 +1298,43 @@ public class TestExportWriter extends SolrTestCaseJ4 { .contains("Must have useDocValuesAsStored='true'")); } + @Test + public void testGlobFields() throws Exception { + assertU(delQ("*:*")); + assertU(commit()); + createLargeIndex(); + SolrQueryRequest req = + req("q", "*:*", "qt", "/export", "fl", "id,*_udvas,*_i_p", "sort", "id asc"); + assertJQ( + req, + "response/numFound==100000", + "response/docs/[0]/id=='0'", + "response/docs/[1]/id=='1'", + "response/docs/[0]/sortabledv_udvas=='0'", + "response/docs/[1]/sortabledv_udvas=='1'", + "response/docs/[0]/small_i_p==0", + "response/docs/[1]/small_i_p==1"); + + assertU(delQ("*:*")); + assertU(commit()); + createLargeIndex(); + req = req("q", "*:*", "qt", "/export", "fl", "*", "sort", "id asc"); + assertJQ( + req, + "response/numFound==100000", + "response/docs/[0]/id=='0'", + "response/docs/[1]/id=='1'", + "response/docs/[0]/sortabledv_udvas=='0'", + "response/docs/[1]/sortabledv_udvas=='1'", + "response/docs/[0]/small_i_p==0", + "response/docs/[1]/small_i_p==1"); + + String jq = JQ(req); + assertFalse( + "Fields without docvalues and useDocValuesAsStored should not be returned", + jq.contains("\"sortabledv\"")); + } + @SuppressWarnings("rawtypes") private void validateSort(int numDocs) throws Exception { // 10 fields
