This is an automated email from the ASF dual-hosted git repository. epugh pushed a commit to branch branch_9x in repository https://gitbox.apache.org/repos/asf/solr.git
commit 870b1d8a22813d313079bc2497e4c64529f8914f Author: Eric Pugh <[email protected]> AuthorDate: Mon Jul 31 07:43:33 2023 -0400 SOLR-16883: Use postlogs tool from windows and unix via solr cli infrastructure. (#1786) Postlogs tool only works on Linux, and was it's own implementation of a command line tool, with different patterns then the other tools. Migrate over to "bin/solr postlogs" and it now works like other CLI tools, including working on Windows. --- solr/CHANGES.txt | 4 +- solr/bin/postlogs | 1 + solr/bin/solr | 2 +- .../{SolrLogPostTool.java => PostLogsTool.java} | 69 ++- .../core/src/java/org/apache/solr/cli/SolrCLI.java | 1 + .../java/org/apache/solr/cli/SolrLogPostTool.java | 549 +-------------------- ...rLogPostToolTest.java => PostLogsToolTest.java} | 7 +- solr/packaging/test/test_postlogs.bats | 14 +- .../modules/query-guide/pages/logs.adoc | 40 +- 9 files changed, 104 insertions(+), 583 deletions(-) diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index a80e887b9b6..c7889a598cb 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -15,6 +15,8 @@ Improvements * SOLR-16490: `/admin/cores?action=backupcore` now has a v2 equivalent, available at `GET /api/cores/coreName/backups` (Sanjay Dutt via Jason Gerlowski) +* SOLR-16883: Postlogs tool for indexing Solr logs in Solr now supported on Windows by converting it to a Solr CLI command: `bin/solr postlogs`. `bin/postlogs` script marked deprected. (Eric Pugh, Will White) + Optimizations --------------------- @@ -179,7 +181,7 @@ Improvements * SOLR-16687: Add support of SolrClassLoader to SolrZkClient (Lamine Idjeraoui via Jason Gerlowski & Houston Putman) -* SOLR-9378: Internal shard requests no longer include the wasteful shard.url param. [shard] transformer now defaults to returning +* SOLR-9378: Internal shard requests no longer include the wasteful shard.url param. [shard] transformer now defaults to returning only the shard id (based on luceneMatchVersion), but can be configured to return the legacy list of replicas. (hossman) * SOLR-16816: Update node metrics while making affinityPlacement selections. Therefore selections can be made given the expected cluster diff --git a/solr/bin/postlogs b/solr/bin/postlogs index 71a5e7a255d..249f693230d 100755 --- a/solr/bin/postlogs +++ b/solr/bin/postlogs @@ -31,6 +31,7 @@ # ############################################################################################ +echo "This script has been deprecated in favour of 'bin/solr postlogs' command." SOLR_TIP="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )"/.. java -classpath "$SOLR_TIP/server/lib/ext/*:$SOLR_TIP/server/solr-webapp/webapp/WEB-INF/lib/*" org.apache.solr.cli.SolrLogPostTool $1 $2 diff --git a/solr/bin/solr b/solr/bin/solr index 9183d840db0..44408103603 100644 --- a/solr/bin/solr +++ b/solr/bin/solr @@ -940,7 +940,7 @@ if [ $# -eq 1 ]; then -help|-h) print_usage "" exit - ;; + ;; esac fi diff --git a/solr/core/src/java/org/apache/solr/cli/SolrLogPostTool.java b/solr/core/src/java/org/apache/solr/cli/PostLogsTool.java similarity index 93% copy from solr/core/src/java/org/apache/solr/cli/SolrLogPostTool.java copy to solr/core/src/java/org/apache/solr/cli/PostLogsTool.java index 8ee87a3b8a8..91e1a4e6fbf 100644 --- a/solr/core/src/java/org/apache/solr/cli/SolrLogPostTool.java +++ b/solr/core/src/java/org/apache/solr/cli/PostLogsTool.java @@ -14,11 +14,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.solr.cli; import java.io.BufferedReader; import java.io.IOException; import java.io.LineNumberReader; +import java.io.PrintStream; import java.net.URLDecoder; import java.nio.charset.Charset; import java.nio.file.Files; @@ -32,6 +34,8 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; import java.util.stream.Stream; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Option; import org.apache.solr.client.solrj.SolrClient; import org.apache.solr.client.solrj.impl.Http2SolrClient; import org.apache.solr.client.solrj.request.UpdateRequest; @@ -40,29 +44,48 @@ import org.apache.solr.common.SolrInputField; import org.apache.solr.handler.component.ShardRequest; /** A command line tool for indexing Solr logs in the out-of-the-box log format. */ -public class SolrLogPostTool { - - public static void main(String[] args) throws Exception { - - if (args.length != 2) { - CLIO.out(""); - CLIO.out("postlogs is a simple tool for indexing Solr logs."); - CLIO.out(""); - CLIO.out("parameters:"); - CLIO.out(""); - CLIO.out("-- baseUrl: Example http://localhost:8983/solr/collection1";); - CLIO.out("-- rootDir: All files found at or below the root will be indexed."); - CLIO.out(""); - CLIO.out( - "Sample syntax 1: ./bin/postlogs http://localhost:8983/solr/collection1 /user/foo/logs/solr.log"); - CLIO.out( - "Sample syntax 2: ./bin/postlogs http://localhost:8983/solr/collection1 /user/foo/logs"); - CLIO.out(""); - return; - } +public class PostLogsTool extends ToolBase { + + public PostLogsTool() { + this(CLIO.getOutStream()); + } + + public PostLogsTool(PrintStream stdout) { + super(stdout); + } + + @Override + public String getName() { + return "postlogs"; + } + + @Override + public List<Option> getOptions() { + return List.of( + Option.builder("url") + .longOpt("url") + .argName("ADDRESS") + .hasArg() + .required(true) + .desc("Address of the collection, example http://localhost:8983/solr/collection1/.";) + .build(), + Option.builder("rootdir") + .longOpt("rootdir") + .argName("DIRECTORY") + .hasArg() + .required(true) + .desc("All files found at or below the root directory will be indexed.") + .build()); + } + + @Override + public void runImpl(CommandLine cli) throws Exception { + String url = cli.getOptionValue("url"); + String rootDir = cli.getOptionValue("rootdir"); + runCommand(url, rootDir); + } - String baseUrl = args[0]; - String root = args[1]; + public void runCommand(String baseUrl, String root) throws IOException { Http2SolrClient.Builder builder = new Http2SolrClient.Builder(baseUrl); try (SolrClient client = builder.build()) { @@ -114,7 +137,7 @@ public class SolrLogPostTool { } } - private static void sendBatch(SolrClient client, UpdateRequest request, boolean lastRequest) { + private void sendBatch(SolrClient client, UpdateRequest request, boolean lastRequest) { final String beginMessage = lastRequest ? "Sending last batch ..." : "Sending batch of 300 log records..."; CLIO.out(beginMessage); diff --git a/solr/core/src/java/org/apache/solr/cli/SolrCLI.java b/solr/core/src/java/org/apache/solr/cli/SolrCLI.java index b0288f2f870..ce1ea16d564 100755 --- a/solr/core/src/java/org/apache/solr/cli/SolrCLI.java +++ b/solr/core/src/java/org/apache/solr/cli/SolrCLI.java @@ -240,6 +240,7 @@ public class SolrCLI implements CLIO { else if ("auth".equals(toolType)) return new AuthTool(); else if ("export".equals(toolType)) return new ExportTool(); else if ("package".equals(toolType)) return new PackageTool(); + else if ("postlogs".equals(toolType)) return new PostLogsTool(); else if ("version".equals(toolType)) return new VersionTool(); // If you add a built-in tool to this class, add it here to avoid diff --git a/solr/core/src/java/org/apache/solr/cli/SolrLogPostTool.java b/solr/core/src/java/org/apache/solr/cli/SolrLogPostTool.java index 8ee87a3b8a8..cf4fee63b35 100644 --- a/solr/core/src/java/org/apache/solr/cli/SolrLogPostTool.java +++ b/solr/core/src/java/org/apache/solr/cli/SolrLogPostTool.java @@ -16,30 +16,12 @@ */ package org.apache.solr.cli; -import java.io.BufferedReader; -import java.io.IOException; -import java.io.LineNumberReader; -import java.net.URLDecoder; -import java.nio.charset.Charset; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.TreeMap; -import java.util.UUID; -import java.util.regex.Matcher; -import java.util.regex.Pattern; -import java.util.stream.Collectors; -import java.util.stream.Stream; -import org.apache.solr.client.solrj.SolrClient; -import org.apache.solr.client.solrj.impl.Http2SolrClient; -import org.apache.solr.client.solrj.request.UpdateRequest; -import org.apache.solr.common.SolrInputDocument; -import org.apache.solr.common.SolrInputField; -import org.apache.solr.handler.component.ShardRequest; - -/** A command line tool for indexing Solr logs in the out-of-the-box log format. */ +/** + * A command line tool for indexing Solr logs in the out-of-the-box log format. + * + * @deprecated Please use {@link PostLogsTool} that is exposed as 'bin/solr postlogs'. + */ +@Deprecated(since = "9.4") public class SolrLogPostTool { public static void main(String[] args) throws Exception { @@ -60,524 +42,9 @@ public class SolrLogPostTool { CLIO.out(""); return; } - String baseUrl = args[0]; String root = args[1]; - - Http2SolrClient.Builder builder = new Http2SolrClient.Builder(baseUrl); - try (SolrClient client = builder.build()) { - int rec = 0; - UpdateRequest request = new UpdateRequest(); - - List<Path> files; - try (Stream<Path> stream = Files.walk(Path.of(root), Integer.MAX_VALUE)) { - files = stream.filter(Files::isRegularFile).collect(Collectors.toList()); - } - - for (Path file : files) { - try (LineNumberReader reader = - new LineNumberReader(Files.newBufferedReader(file, Charset.defaultCharset()))) { - LogRecordReader recordReader = new LogRecordReader(reader); - SolrInputDocument doc; - String fileName = file.getFileName().toString(); - while (true) { - try { - doc = recordReader.readRecord(); - } catch (Throwable t) { - CLIO.err( - "Error reading log record:" + reader.getLineNumber() + " from file:" + fileName); - CLIO.err(t.getMessage()); - continue; - } - - if (doc == null) { - break; - } - - rec++; - UUID id = UUID.randomUUID(); - doc.setField("id", id.toString()); - doc.setField("file_s", fileName); - request.add(doc); - if (rec == 300) { - sendBatch(client, request, false /* normal batch */); - request = new UpdateRequest(); - rec = 0; - } - } - } - } - - if (rec > 0) { - sendBatch(client, request, true /* last batch */); - } - } - } - - private static void sendBatch(SolrClient client, UpdateRequest request, boolean lastRequest) { - final String beginMessage = - lastRequest ? "Sending last batch ..." : "Sending batch of 300 log records..."; - CLIO.out(beginMessage); - try { - request.process(client); - CLIO.out("Batch sent"); - } catch (Exception e) { - CLIO.err("Batch sending failed: " + e.getMessage()); - e.printStackTrace(CLIO.getErrStream()); - } - - if (lastRequest) { - try { - client.commit(); - CLIO.out("Committed"); - } catch (Exception e) { - CLIO.err("Unable to commit documents: " + e.getMessage()); - e.printStackTrace(CLIO.getErrStream()); - } - } - } - - public static class LogRecordReader { - - private BufferedReader bufferedReader; - private String pushedBack = null; - private boolean finished = false; - private String cause; - private Pattern p = - Pattern.compile("^(\\d\\d\\d\\d\\-\\d\\d\\-\\d\\d[\\s|T]\\d\\d:\\d\\d\\:\\d\\d.\\d\\d\\d)"); - private Pattern minute = - Pattern.compile("^(\\d\\d\\d\\d\\-\\d\\d\\-\\d\\d[\\s|T]\\d\\d:\\d\\d)"); - private Pattern tenSecond = - Pattern.compile("^(\\d\\d\\d\\d\\-\\d\\d\\-\\d\\d[\\s|T]\\d\\d:\\d\\d:\\d)"); - - public LogRecordReader(BufferedReader bufferedReader) throws IOException { - this.bufferedReader = bufferedReader; - } - - public SolrInputDocument readRecord() throws IOException { - while (true) { - String line = null; - - if (finished) { - return null; - } - - if (pushedBack != null) { - line = pushedBack; - pushedBack = null; - } else { - line = bufferedReader.readLine(); - } - - if (line != null) { - SolrInputDocument lineDoc = new SolrInputDocument(); - String date = parseDate(line); - String minute = parseMinute(line); - String tenSecond = parseTenSecond(line); - lineDoc.setField("date_dt", date); - lineDoc.setField("time_minute_s", minute); - lineDoc.setField("time_ten_second_s", tenSecond); - lineDoc.setField("line_t", line); - lineDoc.setField("type_s", "other"); // Overridden by known types below - - if (line.contains("Registered new searcher")) { - parseNewSearch(lineDoc, line); - } else if (line.contains("path=/update")) { - parseUpdate(lineDoc, line); - } else if (line.contains(" ERROR ")) { - this.cause = null; - parseError(lineDoc, line, readTrace()); - } else if (line.contains("QTime=")) { - parseQueryRecord(lineDoc, line); - } - - return lineDoc; - } else { - return null; - } - } - } - - private String readTrace() throws IOException { - StringBuilder buf = new StringBuilder(); - buf.append("%html "); - - while (true) { - String line = bufferedReader.readLine(); - if (line == null) { - finished = true; - return buf.toString(); - } else { - // look for a date at the beginning of the line - // If it's not there then read into the stack trace buffer - Matcher m = p.matcher(line); - - if (!m.find() && buf.length() < 10000) { - // Line does not start with a timestamp so append to the stack trace - buf.append(line.replace("\t", " ")).append("<br/>"); - if (line.startsWith("Caused by:")) { - this.cause = line; - } - } else { - pushedBack = line; - break; - } - } - } - - return buf.toString(); - } - - private String parseDate(String line) { - Matcher m = p.matcher(line); - if (m.find()) { - String date = m.group(1); - return date.replace(" ", "T") + "Z"; - } - - return null; - } - - private String parseMinute(String line) { - Matcher m = minute.matcher(line); - if (m.find()) { - String date = m.group(1); - return date.replace(" ", "T") + ":00Z"; - } - - return null; - } - - private String parseTenSecond(String line) { - Matcher m = tenSecond.matcher(line); - if (m.find()) { - String date = m.group(1); - return date.replace(" ", "T") + "0Z"; - } - - return null; - } - - private void setFieldIfUnset(SolrInputDocument doc, String fieldName, String fieldValue) { - if (doc.containsKey(fieldName)) return; - - doc.setField(fieldName, fieldValue); - } - - private void parseError(SolrInputDocument lineRecord, String line, String trace) { - lineRecord.setField("type_s", "error"); - - // Don't include traces that have only the %html header. - if (trace != null && trace.length() > 6) { - lineRecord.setField("stack_t", trace); - } - - if (this.cause != null) { - lineRecord.setField("root_cause_t", cause.replace("Caused by:", "").trim()); - } - - lineRecord.setField("collection_s", parseCollection(line)); - lineRecord.setField("core_s", parseCore(line)); - lineRecord.setField("shard_s", parseShard(line)); - lineRecord.setField("replica_s", parseReplica(line)); - } - - private void parseQueryRecord(SolrInputDocument lineRecord, String line) { - lineRecord.setField("qtime_i", parseQTime(line)); - lineRecord.setField("status_s", parseStatus(line)); - - String path = parsePath(line); - lineRecord.setField("path_s", path); - - if (line.contains("hits=")) { - lineRecord.setField("hits_l", parseHits(line)); - } - - String params = parseParams(line); - lineRecord.setField("params_t", params); - addParams(lineRecord, params); - - lineRecord.setField("collection_s", parseCollection(line)); - lineRecord.setField("core_s", parseCore(line)); - lineRecord.setField("node_s", parseNode(line)); - lineRecord.setField("shard_s", parseShard(line)); - lineRecord.setField("replica_s", parseReplica(line)); - - if (path != null && path.contains("/admin")) { - lineRecord.setField("type_s", "admin"); - } else if (path != null && params.contains("/replication")) { - lineRecord.setField("type_s", "replication"); - } else if (path != null && path.contains("/get")) { - lineRecord.setField("type_s", "get"); - } else { - lineRecord.setField("type_s", "query"); - } - } - - private void parseNewSearch(SolrInputDocument lineRecord, String line) { - lineRecord.setField("core_s", parseCore(line)); - lineRecord.setField("type_s", "newSearcher"); - lineRecord.setField("collection_s", parseCollection(line)); - lineRecord.setField("shard_s", parseShard(line)); - lineRecord.setField("replica_s", parseReplica(line)); - } - - private String parseCollection(String line) { - char[] ca = {' ', ']', ','}; - String[] parts = line.split("c:"); - if (parts.length >= 2) { - return readUntil(parts[1], ca); - } else { - return null; - } - } - - private void parseUpdate(SolrInputDocument lineRecord, String line) { - if (line.contains("deleteByQuery=")) { - lineRecord.setField("type_s", "deleteByQuery"); - } else if (line.contains("delete=")) { - lineRecord.setField("type_s", "delete"); - } else if (line.contains("commit=true")) { - lineRecord.setField("type_s", "commit"); - } else { - lineRecord.setField("type_s", "update"); - } - - lineRecord.setField("collection_s", parseCollection(line)); - lineRecord.setField("core_s", parseCore(line)); - lineRecord.setField("shard_s", parseShard(line)); - lineRecord.setField("replica_s", parseReplica(line)); - } - - private String parseCore(String line) { - char[] ca = {' ', ']', '}', ','}; - String[] parts = line.split("x:"); - if (parts.length >= 2) { - return readUntil(parts[1], ca); - } else { - return null; - } - } - - private String parseShard(String line) { - char[] ca = {' ', ']', '}', ','}; - String[] parts = line.split("s:"); - if (parts.length >= 2) { - return readUntil(parts[1], ca); - } else { - return null; - } - } - - private String parseReplica(String line) { - char[] ca = {' ', ']', '}', ','}; - String[] parts = line.split("r:"); - if (parts.length >= 2) { - return readUntil(parts[1], ca); - } else { - return null; - } - } - - private String parsePath(String line) { - char[] ca = {' '}; - String[] parts = line.split(" path="); - if (parts.length == 2) { - return readUntil(parts[1], ca); - } else { - return null; - } - } - - private String parseQTime(String line) { - char[] ca = {'\n', '\r'}; - String[] parts = line.split(" QTime="); - if (parts.length == 2) { - return readUntil(parts[1], ca); - } else { - return null; - } - } - - private String parseNode(String line) { - char[] ca = {' ', ']', '}', ','}; - String[] parts = line.split("node_name=n:"); - if (parts.length >= 2) { - return readUntil(parts[1], ca); - } else { - return null; - } - } - - private String parseStatus(String line) { - char[] ca = {' ', '\n', '\r'}; - String[] parts = line.split(" status="); - if (parts.length == 2) { - return readUntil(parts[1], ca); - } else { - return null; - } - } - - private String parseHits(String line) { - char[] ca = {' '}; - String[] parts = line.split(" hits="); - if (parts.length == 2) { - return readUntil(parts[1], ca); - } else { - return null; - } - } - - private String parseParams(String line) { - char[] ca = {' '}; - String[] parts = line.split(" params="); - if (parts.length == 2) { - String p = readUntil(parts[1].substring(1), ca); - return p.substring(0, p.length() - 1); - } else { - return null; - } - } - - private String readUntil(String s, char[] chars) { - StringBuilder builder = new StringBuilder(); - for (int i = 0; i < s.length(); i++) { - char a = s.charAt(i); - for (char c : chars) { - if (a == c) { - return builder.toString(); - } - } - builder.append(a); - } - - return builder.toString(); - } - - private void addParams(SolrInputDocument doc, String params) { - String[] pairs = params.split("&"); - for (String pair : pairs) { - String[] parts = pair.split("="); - if (parts.length == 2 && parts[0].equals("q")) { - String dq = URLDecoder.decode(parts[1], Charset.defaultCharset()); - setFieldIfUnset(doc, "q_s", dq); - setFieldIfUnset(doc, "q_t", dq); - } - - if (parts[0].equals("rows")) { - String dr = URLDecoder.decode(parts[1], Charset.defaultCharset()); - setFieldIfUnset(doc, "rows_i", dr); - } - - if (parts[0].equals("start")) { - String dr = URLDecoder.decode(parts[1], Charset.defaultCharset()); - setFieldIfUnset(doc, "start_i", dr); - } - - if (parts[0].equals("distrib")) { - String dr = URLDecoder.decode(parts[1], Charset.defaultCharset()); - setFieldIfUnset(doc, "distrib_s", dr); - } - - if (parts[0].equals("shards")) { - setFieldIfUnset(doc, "shards_s", "true"); - } - - if (parts[0].equals("ids") && !isRTGRequest(doc)) { - setFieldIfUnset(doc, "ids_s", "true"); - } - - if (parts[0].equals("isShard")) { - String dr = URLDecoder.decode(parts[1], Charset.defaultCharset()); - setFieldIfUnset(doc, "isShard_s", dr); - } - - if (parts[0].equals("wt")) { - String dr = URLDecoder.decode(parts[1], Charset.defaultCharset()); - setFieldIfUnset(doc, "wt_s", dr); - } - - if (parts[0].equals("facet")) { - String dr = URLDecoder.decode(parts[1], Charset.defaultCharset()); - setFieldIfUnset(doc, "facet_s", dr); - } - - if (parts[0].equals("shards.purpose")) { - try { - int purpose = Integer.parseInt(parts[1]); - String[] purposes = getRequestPurposeNames(purpose); - for (String p : purposes) { - doc.addField("purpose_ss", p); - } - } catch (Throwable e) { - // We'll just sit on this for now and not interrupt the load for this one field. - } - } - } - - // Special params used to determine what stage a query is. - // So we populate with defaults. - // The absence of the distrib params means it's a distributed query. - setFieldIfUnset(doc, "distrib_s", "true"); - setFieldIfUnset(doc, "shards_s", "false"); - setFieldIfUnset(doc, "ids_s", "false"); - } - - private boolean isRTGRequest(SolrInputDocument doc) { - final SolrInputField path = doc.getField("path_s"); - if (path == null) return false; - - return "/get".equals(path.getValue()); - } - } - - private static final Map<Integer, String> purposes; - protected static final String UNKNOWN_VALUE = "Unknown"; - private static final String[] purposeUnknown = new String[] {UNKNOWN_VALUE}; - - public static String[] getRequestPurposeNames(Integer reqPurpose) { - if (reqPurpose != null) { - int valid = 0; - for (Map.Entry<Integer, String> entry : purposes.entrySet()) { - if ((reqPurpose & entry.getKey()) != 0) { - valid++; - } - } - if (valid == 0) { - return purposeUnknown; - } else { - String[] result = new String[valid]; - int i = 0; - for (Map.Entry<Integer, String> entry : purposes.entrySet()) { - if ((reqPurpose & entry.getKey()) != 0) { - result[i] = entry.getValue(); - i++; - } - } - return result; - } - } - return purposeUnknown; - } - - static { - Map<Integer, String> map = new TreeMap<>(); - map.put(ShardRequest.PURPOSE_PRIVATE, "PRIVATE"); - map.put(ShardRequest.PURPOSE_GET_TOP_IDS, "GET_TOP_IDS"); - map.put(ShardRequest.PURPOSE_REFINE_TOP_IDS, "REFINE_TOP_IDS"); - map.put(ShardRequest.PURPOSE_GET_FACETS, "GET_FACETS"); - map.put(ShardRequest.PURPOSE_REFINE_FACETS, "REFINE_FACETS"); - map.put(ShardRequest.PURPOSE_GET_FIELDS, "GET_FIELDS"); - map.put(ShardRequest.PURPOSE_GET_HIGHLIGHTS, "GET_HIGHLIGHTS"); - map.put(ShardRequest.PURPOSE_GET_DEBUG, "GET_DEBUG"); - map.put(ShardRequest.PURPOSE_GET_STATS, "GET_STATS"); - map.put(ShardRequest.PURPOSE_GET_TERMS, "GET_TERMS"); - map.put(ShardRequest.PURPOSE_GET_TOP_GROUPS, "GET_TOP_GROUPS"); - map.put(ShardRequest.PURPOSE_GET_MLT_RESULTS, "GET_MLT_RESULTS"); - map.put(ShardRequest.PURPOSE_REFINE_PIVOT_FACETS, "REFINE_PIVOT_FACETS"); - map.put(ShardRequest.PURPOSE_SET_TERM_STATS, "SET_TERM_STATS"); - map.put(ShardRequest.PURPOSE_GET_TERM_STATS, "GET_TERM_STATS"); - purposes = Collections.unmodifiableMap(map); + PostLogsTool postLogsTool = new PostLogsTool(); + postLogsTool.runCommand(baseUrl, root); } } diff --git a/solr/core/src/test/org/apache/solr/cli/SolrLogPostToolTest.java b/solr/core/src/test/org/apache/solr/cli/PostLogsToolTest.java similarity index 98% rename from solr/core/src/test/org/apache/solr/cli/SolrLogPostToolTest.java rename to solr/core/src/test/org/apache/solr/cli/PostLogsToolTest.java index 2ebfb9f1b82..6240c5443b1 100644 --- a/solr/core/src/test/org/apache/solr/cli/SolrLogPostToolTest.java +++ b/solr/core/src/test/org/apache/solr/cli/PostLogsToolTest.java @@ -22,20 +22,19 @@ import java.util.ArrayList; import java.util.Collection; import java.util.List; import org.apache.solr.SolrTestCaseJ4; -import org.apache.solr.cli.SolrLogPostTool.LogRecordReader; +import org.apache.solr.cli.PostLogsTool.LogRecordReader; import org.apache.solr.common.SolrInputDocument; import org.apache.solr.common.SolrInputField; import org.junit.BeforeClass; import org.junit.Test; -public class SolrLogPostToolTest extends SolrTestCaseJ4 { +public class PostLogsToolTest extends SolrTestCaseJ4 { private static boolean solr9Format; @BeforeClass public static void beforeClass() { solr9Format = random().nextBoolean(); - System.out.println("Solr 9 Format: " + solr9Format); } private String sometimesSolr9Format(String record) { @@ -310,7 +309,7 @@ public class SolrLogPostToolTest extends SolrTestCaseJ4 { assertEquals(collection.getValue(), "test"); } - // Ensure SolrLogPostTool parses _all_ log lines into searchable records + // Ensure PostLogsTool parses _all_ log lines into searchable records @Test public void testOtherRecord() throws Exception { final String record = diff --git a/solr/packaging/test/test_postlogs.bats b/solr/packaging/test/test_postlogs.bats index 32f7c379a7c..64c8230fefd 100644 --- a/solr/packaging/test/test_postlogs.bats +++ b/solr/packaging/test/test_postlogs.bats @@ -38,7 +38,7 @@ teardown() { delete_all_collections } -@test "post solr log into solr" { +@test "post solr log into solr via script" { run solr create_collection -c COLL_NAME assert_output --partial "Created collection 'COLL_NAME'" @@ -49,3 +49,15 @@ teardown() { run curl 'http://localhost:8983/solr/COLL_NAME/select?q=*:*' refute_output --partial '"numFound":0' } + +@test "post solr log into solr via cli" { + run solr create_collection -c COLL_NAME + assert_output --partial "Created collection 'COLL_NAME'" + + run solr postlogs -url http://localhost:8983/solr/COLL_NAME -rootdir ${SOLR_LOGS_DIR}/solr.log + assert_output --partial 'Sending last batch' + assert_output --partial 'Committed' + + run curl 'http://localhost:8983/solr/COLL_NAME/select?q=*:*' + refute_output --partial '"numFound":0' +} diff --git a/solr/solr-ref-guide/modules/query-guide/pages/logs.adoc b/solr/solr-ref-guide/modules/query-guide/pages/logs.adoc index 7b9412570bd..7ac02556335 100644 --- a/solr/solr-ref-guide/modules/query-guide/pages/logs.adoc +++ b/solr/solr-ref-guide/modules/query-guide/pages/logs.adoc @@ -24,25 +24,41 @@ See the xref:math-start.adoc[] chapter to learn how to get started with visualiz == Loading -The out-of-the-box Solr log format can be loaded into a Solr index using the `bin/postlogs` command line tool located in the `bin/` directory of the Solr distribution. +The out-of-the-box Solr log format can be loaded into a Solr index using the `bin/solr postlogs` command line tool located in the `bin/` directory of the Solr distribution. -NOTE: If working from the source distribution the -distribution must first be built before `postlogs` can be run. +=== Postlogs -The `postlogs` script is designed to be run from the root directory of the Solr distribution. +The `postlogs` command reads in Solr's log format and indexes it in a Solr collection. -The `postlogs` script takes two parameters: +`bin/solr postlogs [options]` -* Solr base URL (with collection): `http://localhost:8983/solr/logs` -* File path to root of the logs directory: All files found under this directory (including sub-directories) will be indexed. +`bin/solr postlogs -help` + +==== Healthcheck Parameters + +`-url <ADDRESS>`:: ++ +[%autowidth,frame=none] +|=== +|Required |Default: none +|=== ++ +Address of the collection, example http://localhost:8983/solr/collection1/. ++ + +`-rootdir <DIRECTORY>`:: ++ +[%autowidth,frame=none] +|=== +|Required |Default: none +|=== ++ +File path to root of the logs directory: All files found under this directory (including sub-directories) will be indexed. If the path points to a single log file only that log file will be loaded. -Below is a sample execution of the `postlogs` tool: ++ +*Example*: `bin/solr postlogs --url http://localhost:8983/solr/logs --rootdir /var/logs/solrlogs` -[source,text] ----- -./bin/postlogs http://localhost:8983/solr/logs /var/logs/solrlogs ----- The example above will index all the log files under `/var/logs/solrlogs` to the `logs` collection found at the base url `http://localhost:8983/solr`.
