Builds on jenkins are failing due to precommit failures in this code...
> On 20 Feb 2018, at 09:47, [email protected] wrote:
>
> Repository: lucene-solr
> Updated Branches:
> refs/heads/master dfc0fe86e -> 4bfcbc5c6
>
>
> http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4bfcbc5c/solr/contrib/prometheus-exporter/src/test/org/apache/solr/prometheus/scraper/config/SolrQueryConfigTest.java
> ----------------------------------------------------------------------
> diff --git
> a/solr/contrib/prometheus-exporter/src/test/org/apache/solr/prometheus/scraper/config/SolrQueryConfigTest.java
>
> b/solr/contrib/prometheus-exporter/src/test/org/apache/solr/prometheus/scraper/config/SolrQueryConfigTest.java
> new file mode 100644
> index 0000000..c62d354
> --- /dev/null
> +++
> b/solr/contrib/prometheus-exporter/src/test/org/apache/solr/prometheus/scraper/config/SolrQueryConfigTest.java
> @@ -0,0 +1,121 @@
> +/*
> + * Licensed to the Apache Software Foundation (ASF) under one or more
> + * contributor license agreements. See the NOTICE file distributed with
> + * this work for additional information regarding copyright ownership.
> + * The ASF licenses this file to You under the Apache License, Version 2.0
> + * (the "License"); you may not use this file except in compliance with
> + * the License. You may obtain a copy of the License at
> + *
> + * http://www.apache.org/licenses/LICENSE-2.0
> + *
> + * Unless required by applicable law or agreed to in writing, software
> + * distributed under the License is distributed on an "AS IS" BASIS,
> + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
> + * See the License for the specific language governing permissions and
> + * limitations under the License.
> + */
> +package org.apache.solr.prometheus.scraper.config;
> +
> +import org.apache.solr.SolrTestCaseJ4;
> +import org.junit.Test;
> +
> +import java.util.ArrayList;
> +import java.util.Arrays;
> +import java.util.LinkedHashMap;
> +import java.util.List;
> +
> +/**
> + * Unit test for SolrQueryConfig.
> + */
> +public class SolrQueryConfigTest extends SolrTestCaseJ4 {
> + @Test
> + public void testQueryConfig() throws Exception {
> + SolrQueryConfig queryConfig = new SolrQueryConfig();
> +
> + assertNotNull(queryConfig);
> + }
> +
> + @Test
> + public void testGetCollection() throws Exception {
> + SolrQueryConfig queryConfig = new SolrQueryConfig();
> +
> + String expected = "";
> + String actual = queryConfig.getCollection();
> + assertEquals(expected, actual);
> + }
> +
> + @Test
> + public void testSetCollection() throws Exception {
> + SolrQueryConfig queryConfig = new SolrQueryConfig();
> +
> + queryConfig.setCollection("collection1");
> +
> + String expected = "collection1";
> + String actual = queryConfig.getCollection();
> + assertEquals(expected, actual);
> + }
> +
> + @Test
> + public void testGetPath() throws Exception {
> + SolrQueryConfig queryConfig = new SolrQueryConfig();
> +
> + String expected = "";
> + String actual = queryConfig.getPath();
> + assertEquals(expected, actual);
> + }
> +
> + @Test
> + public void testSetPath() throws Exception {
> + SolrQueryConfig queryConfig = new SolrQueryConfig();
> +
> + queryConfig.setPath("/select");
> +
> + String expected = "/select";
> + String actual = queryConfig.getPath();
> + assertEquals(expected, actual);
> + }
> +
> + @Test
> + public void testGetParams() throws Exception {
> + SolrQueryConfig queryConfig = new SolrQueryConfig();
> +
> + List<LinkedHashMap<String, String>> expected = new ArrayList<>();
> + List<LinkedHashMap<String, String>> actual = queryConfig.getParams();
> + assertEquals(expected, actual);
> + }
> +
> + @Test
> + public void testSetParams() throws Exception {
> + SolrQueryConfig queryConfig = new SolrQueryConfig();
> +
> + LinkedHashMap<String,String> param1 = new LinkedHashMap<>();
> + param1.put("q", "*:*");
> +
> + LinkedHashMap<String,String> param2 = new LinkedHashMap<>();
> + param2.put("facet", "on");
> +
> + queryConfig.setParams(Arrays.asList(param1, param2));
> +
> + List<LinkedHashMap<String, String>> expected = Arrays.asList(param1,
> param2);
> + List<LinkedHashMap<String, String>> actual = queryConfig.getParams();
> + assertEquals(expected, actual);
> + }
> +
> + @Test
> + public void testGetParamsString() throws Exception {
> + SolrQueryConfig queryConfig = new SolrQueryConfig();
> +
> + LinkedHashMap<String,String> param1 = new LinkedHashMap<>();
> + param1.put("q", "*:*");
> + param1.put("fq", "manu:apple");
> +
> + LinkedHashMap<String,String> param2 = new LinkedHashMap<>();
> + param2.put("facet", "on");
> +
> + queryConfig.setParams(Arrays.asList(param1, param2));
> +
> + String expected = "q=*:*&fq=manu:apple&facet=on";
> + String actual = queryConfig.getParamsString();
> + assertEquals(expected, actual);
> + }
> +}
>
> http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4bfcbc5c/solr/contrib/prometheus-exporter/src/test/org/apache/solr/prometheus/scraper/config/SolrScraperConfigTest.java
> ----------------------------------------------------------------------
> diff --git
> a/solr/contrib/prometheus-exporter/src/test/org/apache/solr/prometheus/scraper/config/SolrScraperConfigTest.java
>
> b/solr/contrib/prometheus-exporter/src/test/org/apache/solr/prometheus/scraper/config/SolrScraperConfigTest.java
> new file mode 100644
> index 0000000..79d1204
> --- /dev/null
> +++
> b/solr/contrib/prometheus-exporter/src/test/org/apache/solr/prometheus/scraper/config/SolrScraperConfigTest.java
> @@ -0,0 +1,86 @@
> +/*
> + * Licensed to the Apache Software Foundation (ASF) under one or more
> + * contributor license agreements. See the NOTICE file distributed with
> + * this work for additional information regarding copyright ownership.
> + * The ASF licenses this file to You under the Apache License, Version 2.0
> + * (the "License"); you may not use this file except in compliance with
> + * the License. You may obtain a copy of the License at
> + *
> + * http://www.apache.org/licenses/LICENSE-2.0
> + *
> + * Unless required by applicable law or agreed to in writing, software
> + * distributed under the License is distributed on an "AS IS" BASIS,
> + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
> + * See the License for the specific language governing permissions and
> + * limitations under the License.
> + */
> +package org.apache.solr.prometheus.scraper.config;
> +
> +import org.apache.solr.prometheus.collector.config.SolrCollectorConfig;
> +import org.apache.solr.SolrTestCaseJ4;
> +import org.junit.Test;
> +import org.yaml.snakeyaml.Yaml;
> +
> +import java.io.FileReader;
> +import java.util.ArrayList;
> +import java.util.List;
> +
> +/**
> + * Unit test for SolrScraperConfig.
> + */
> +public class SolrScraperConfigTest extends SolrTestCaseJ4 {
> + @Test
> + public void testScraperConfig() throws Exception {
> + String configFile = getFile("conf/config.yml").getAbsolutePath();
> +
> + SolrCollectorConfig config = new Yaml().loadAs(new
> FileReader(configFile), SolrCollectorConfig.class);
> +
> + SolrScraperConfig scraperConfig = config.getMetrics();
> +
> + assertNotNull(scraperConfig);
> + }
> +
> + @Test
> + public void testGetJsonQueries() throws Exception {
> + String configFile = getFile("conf/config.yml").getAbsolutePath();
> +
> + SolrCollectorConfig collectorConfig = new Yaml().loadAs(new
> FileReader(configFile), SolrCollectorConfig.class);
> +
> + SolrScraperConfig scraperConfig = collectorConfig.getMetrics();
> +
> + assertNotNull(scraperConfig.getJsonQueries());
> + }
> +
> + @Test
> + public void testSetJsonQueries() throws Exception {
> + List<String> jsonQueries = new ArrayList<>();
> +
> + SolrScraperConfig scraperConfig = new SolrScraperConfig();
> +
> + scraperConfig.setJsonQueries(jsonQueries);
> +
> + assertNotNull(scraperConfig.getJsonQueries());
> + }
> +
> + @Test
> + public void testGetQueryConfig() throws Exception {
> + String configFile = getFile("conf/config.yml").getAbsolutePath();
> +
> + SolrCollectorConfig collectorConfig = new Yaml().loadAs(new
> FileReader(configFile), SolrCollectorConfig.class);
> +
> + SolrScraperConfig scraperConfig = collectorConfig.getMetrics();
> +
> + assertNotNull(scraperConfig.getQuery());
> + }
> +
> + @Test
> + public void testSetQueryConfig() throws Exception {
> + SolrQueryConfig queryConfig = new SolrQueryConfig();
> +
> + SolrScraperConfig scraperConfig = new SolrScraperConfig();
> +
> + scraperConfig.setQuery(queryConfig);
> +
> + assertNotNull(scraperConfig.getQuery());
> + }
> +}
>
> http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4bfcbc5c/solr/solr-ref-guide/src/images/monitoring-solr-with-prometheus-and-grafana/grafana-solr-dashboard.png
> ----------------------------------------------------------------------
> diff --git
> a/solr/solr-ref-guide/src/images/monitoring-solr-with-prometheus-and-grafana/grafana-solr-dashboard.png
>
> b/solr/solr-ref-guide/src/images/monitoring-solr-with-prometheus-and-grafana/grafana-solr-dashboard.png
> new file mode 100644
> index 0000000..69f21a4
> Binary files /dev/null and
> b/solr/solr-ref-guide/src/images/monitoring-solr-with-prometheus-and-grafana/grafana-solr-dashboard.png
> differ
>
> http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4bfcbc5c/solr/solr-ref-guide/src/images/monitoring-solr-with-prometheus-and-grafana/solr-exporter-diagram.png
> ----------------------------------------------------------------------
> diff --git
> a/solr/solr-ref-guide/src/images/monitoring-solr-with-prometheus-and-grafana/solr-exporter-diagram.png
>
> b/solr/solr-ref-guide/src/images/monitoring-solr-with-prometheus-and-grafana/solr-exporter-diagram.png
> new file mode 100644
> index 0000000..96efece
> Binary files /dev/null and
> b/solr/solr-ref-guide/src/images/monitoring-solr-with-prometheus-and-grafana/solr-exporter-diagram.png
> differ
>
> http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4bfcbc5c/solr/solr-ref-guide/src/monitoring-solr-with-prometheus-and-grafana.adoc
> ----------------------------------------------------------------------
> diff --git
> a/solr/solr-ref-guide/src/monitoring-solr-with-prometheus-and-grafana.adoc
> b/solr/solr-ref-guide/src/monitoring-solr-with-prometheus-and-grafana.adoc
> new file mode 100644
> index 0000000..1ac55a2
> --- /dev/null
> +++ b/solr/solr-ref-guide/src/monitoring-solr-with-prometheus-and-grafana.adoc
> @@ -0,0 +1,250 @@
> += Monitoring Solr with Prometheus and Grafana
> +// Licensed to the Apache Software Foundation (ASF) under one
> +// or more contributor license agreements. See the NOTICE file
> +// distributed with this work for additional information
> +// regarding copyright ownership. The ASF licenses this file
> +// to you under the Apache License, Version 2.0 (the
> +// "License"); you may not use this file except in compliance
> +// with the License. You may obtain a copy of the License at
> +//
> +// http://www.apache.org/licenses/LICENSE-2.0
> +//
> +// Unless required by applicable law or agreed to in writing,
> +// software distributed under the License is distributed on an
> +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
> +// KIND, either express or implied. See the License for the
> +// specific language governing permissions and limitations
> +// under the License.
> +
> +You can monitor Solr using solr-exporter that exposes Solr's metrics to
> https://prometheus.io[Prometheus], and visualize metrics using
> https://grafana.com[Grafana].
> +
> +It allows users to monitor not only Solr metrics which come from
> <<metrics-reporting.adoc#metrics-api,Metrics API>> but also facet counts
> which come from <<searching.adoc#searching,Searching>>.
> +
> +.solr-exporter Diagram
> +image::images/monitoring-solr-with-prometheus-and-grafana/solr-exporter-diagram.png[image,width=600]
> +
> +This feature is experimental status.
> +
> +== Running solr-exporter
> +
> +You can start solr-exporter by running `./bin/solr-exporter` from the
> solr-exporter directory.
> +
> +[source,plain]
> +----
> +$ cd ./contrib/prometheus-exporter
> +$ ./bin/solr-exporter -p 9983 -b http://localhost:8983/solr -f
> ./conf/config.yml -n 8
> +----
> +
> +If you are on Windows platform, you can start solr-exporter by running
> `.\bin\solr-exporter.cmd` instead.
> +
> +[source,plain]
> +----
> +> cd .\contrib\prometheus
> +> .\bin\solr-exporter.cmd -p 9983 -b http://localhost:8983/solr -f
> .\conf\config.yml -n 8
> +----
> +
> +You can also connect to Solr in SolrCloud mode like this.
> +
> +[source,plain]
> +----
> +$ cd ./contrib/prometheus
> +$ ./bin/solr-exporter -p 9983 -z localhost:2181/solr -f ./conf/config.yml -n
> 16
> +----
> +
> +See command help:
> +
> +[source,plain]
> +----
> +$ ./bin/solr-exporter -h
> +usage: SolrCollector [-h] [-v] [-p PORT] [-b BASE_URL] [-z ZK_HOST] [-f
> CONFIG]
> + [-n NUM_THREADS]
> +
> +Prometheus exporter for Apache Solr.
> +
> +optional arguments:
> + -h, --help show this help message and exit
> + -p PORT, --port PORT solr-exporter listen port
> + -b BASE_URL, --baseurl BASE_URL
> + specify Solr base URL when connecting to Solr in
> standalone mode (for
> + example 'http://localhost:8983/solr')
> + -z ZK_HOST, --zkhost ZK_HOST
> + specify ZooKeeper connection string when
> connecting to Solr in
> + SolrCloud mode (for example 'localhost:2181/solr')
> + -f CONFIG, --config-file CONFIG
> + specify configuration file
> + -n NUM_THREADS, --num-thread NUM_THREADS
> + specify number of threads
> +----
> +
> +The Solr's metrics exposed by solr-exporter can see at the following URL.
> +
> +http://localhost:9983/metrics[http://localhost:9983/metrics]
> +
> +
> +== Configuration
> +
> +The configuration is in `./config/config.yml`. An example with all possible
> options:
> +
> +[source,plain]
> +----
> +ping:
> + query:
> + path: /admin/ping
> + jsonQueries:
> + - |-
> + . as $object | $object |
> + (if $object.status == "OK" then 1.0 else 0.0 end) as $value |
> + {
> + name : "solr_ping",
> + type : "GAUGE",
> + help : "See following URL:
> https://lucene.apache.org/solr/guide/ping.html";,
> + label_names : [],
> + label_values : [],
> + value : $value
> + }
> +
> +metrics:
> + query:
> + path: /admin/metrics
> + params:
> + - group: 'all'
> + - type: 'all'
> + - prefix: ''
> + - property: ''
> + jsonQueries:
> + # solr_metrics_jetty_response_count
> + - |-
> + .metrics["solr.jetty"] | to_entries | .[] | select(.key |
> startswith("org.eclipse.jetty.server.handler.DefaultHandler")) | select(.key
> | endswith("xx-responses")) as $object |
> + $object.key | split(".") | last | split("-") | first as $status |
> + $object.value.count as $value |
> + {
> + name : "solr_metrics_jetty_response_count",
> + type : "gauge",
> + help : "See following URL:
> https://lucene.apache.org/solr/guide/metrics-reporting.html";,
> + label_names : ["status"],
> + label_values : [$status],
> + value : $value
> + }
> +
> +...
> +
> +collections:
> + query:
> + path: /admin/collections
> + params:
> + - action: 'CLUSTERSTATUS'
> + jsonQueries:
> + # solr_collections_cluster_status_live_nodes
> + - |-
> + .cluster.live_nodes | length as $value|
> + {
> + name : "solr_collections_cluster_status_live_nodes",
> + type : "gauge",
> + help : "See following URL:
> https://lucene.apache.org/solr/guide/collections-api.html#clusterstatus";,
> + label_names : [],
> + label_values : [],
> + value : $value
> + }
> +
> +...
> +
> +queries:
> + - query:
> + collection: collection1
> + path: /select
> + params:
> + - q: "*:*"
> + - start: 0
> + - rows: 0
> + - json.facet: |-
> + {
> + category: {
> + type: terms,
> + field: cat
> + }
> + }
> + jsonQueries:
> + # solr_facets_category
> + - |-
> + .facets.category.buckets[] as $object |
> + $object.val as $term |
> + $object.count as $value |
> + {
> + name : "solr_facets_category",
> + type : "gauge",
> + help : "Category facets",
> + label_names : ["term"],
> + label_values : [$term],
> + value : $value
> + }
> +----
> +
> +|===
> +|Name|Description
> +
> +|ping|Scrape <<ping.adoc#ping,Ping>> response.
> +|metrics|Scrape <<metrics-reporting.adoc#metrics-api,Metrics API>> response.
> +|collections|Scrape <<collections-api.adoc#collections-api,Collections API>>
> response.
> +|queries|Scrape <<searching.adoc#searching,Search API>> response.
> +|*.query|Query parameter for each features. You can specify `collection`,
> `core`, `path`, and `params`.
> +|*.jsonQueries|JSON Query that is jq syntax. For more details, see
> https://stedolan.github.io/jq/manual/[https://stedolan.github.io/jq/manual/].
> +|===
> +
> +jq query has to output JSON in the following format.
> +
> +[source,json]
> +----
> +{
> + name : "solr_ping",
> + type : "GAUGE",
> + help : "See following URL:
> https://lucene.apache.org/solr/guide/ping.html";,
> + label_names : ["base_url","core"],
> + label_values : ["http://localhost:8983/solr","collection1";],
> + value : 1.0
> +}
> +----
> +
> +It will be converted to the following exposition format.
> +
> +[source,plain]
> +----
> +# TYPE solr_ping gauge
> +# HELP solr_ping See following URL:
> https://lucene.apache.org/solr/guide/ping.html
> +solr_ping{base_url="http://localhost:8983/solr",core="collection1"} 1.0
> +----
> +
> +|===
> +|Name|Description
> +
> +|name|The metric name to set. For more details, see
> https://prometheus.io/docs/practices/naming/[https://prometheus.io/docs/practices/naming/].
> +|type|The type of the metric, can be `COUNTER`, `GAUGE`, `SUMMARY`,
> `HISTOGRAM` or `UNTYPED`. For more detauils, see
> https://prometheus.io/docs/concepts/metric_types/[https://prometheus.io/docs/concepts/metric_types/].
> +|help|Help text for the metric.
> +|label_names|Label names for the metric. For more details, see
> https://prometheus.io/docs/practices/naming/[https://prometheus.io/docs/practices/naming/].
> +|label_values|Label values for the metric. For more details, see
> https://prometheus.io/docs/practices/naming/[https://prometheus.io/docs/practices/naming/].
> +|value|Value for the metric. Value must be set to Double type.
> +|===
> +
> +
> +== Prometheus Settings
> +
> +You need to specify the solr-exporter listen address into `scrape_configs`
> in `prometheus.yml`. See following example:
> +
> +[source,plain]
> +----
> +scrape_configs:
> + - job_name: 'solr'
> + static_configs:
> + - targets: ['localhost:9983']
> +----
> +
> +When you apply the above settings to prometheus, it will start to pull
> Solr's metrics from solr-exporter.
> +
> +
> +== Grafana Dashboard
> +
> +A Grafana sample dashboard is provided at the following JSON file.
> +
> +`./conf/grafana-solr-dashboard.json`
> +
> +.Grafana Dashboard
> +image::images/monitoring-solr-with-prometheus-and-grafana/grafana-solr-dashboard.png[image,width=800]
>
> http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4bfcbc5c/solr/solr-ref-guide/src/monitoring-solr.adoc
> ----------------------------------------------------------------------
> diff --git a/solr/solr-ref-guide/src/monitoring-solr.adoc
> b/solr/solr-ref-guide/src/monitoring-solr.adoc
> index 0576e53..2fb7077 100644
> --- a/solr/solr-ref-guide/src/monitoring-solr.adoc
> +++ b/solr/solr-ref-guide/src/monitoring-solr.adoc
> @@ -1,5 +1,5 @@
> = Monitoring Solr
> -:page-children: metrics-reporting, mbean-request-handler,
> configuring-logging, using-jmx-with-solr, performance-statistics-reference
> +:page-children: metrics-reporting, mbean-request-handler,
> configuring-logging, using-jmx-with-solr,
> monitoring-solr-with-prometheus-and-grafana, performance-statistics-reference
> // Licensed to the Apache Software Foundation (ASF) under one
> // or more contributor license agreements. See the NOTICE file
> // distributed with this work for additional information
> @@ -30,6 +30,8 @@ Common administrative tasks include:
>
> <<using-jmx-with-solr.adoc#using-jmx-with-solr,Using JMX with Solr>>:
> Describes how to use Java Management Extensions with Solr.
>
> +<<monitoring-solr-with-prometheus-and-grafana.adoc#monitoring-solr-with-prometheus-and-grafana,Monitoring
> Solr with Prometheus and Grafana>>: Describes how to monitor Solr with
> Prometheus and Grafana.
> +
> <<performance-statistics-reference.adoc#performance-statistics-reference,Performance
> Statistics Reference>>: Additional information on statistics returned from
> JMX.
>
>
>
