SOLR-18147 Make a new Grafana dashboard for Solr 10.x

.gitattributes

@@ -5,6 +5,10 @@ versions.lock text eol=lf
 # Gradle files are always in LF.
 *.gradle          text eol=lf
 
+# Monitoring mixin generated outputs — always LF to ensure make regeneration is idempotent.
+solr/monitoring/grafana-solr-dashboard.json text eol=lf
+solr/monitoring/prometheus-solr-alerts.yml  text eol=lf
+
 # Exclude from source release tarball (git archive).  Avoids ASF license header requirements.
 /.*               export-ignore
 /dev-docs         export-ignore

changelog/unreleased/GITHUB4210-metrics-accept-header.yml

@@ -0,0 +1,14 @@
+# See https://github.com/apache/solr/blob/main/dev-docs/changelog.adoc
+
+title: >
+  The v2 `/api/metrics` endpoint no longer rejects standard `Accept` headers. It compared the whole
+  header against a single content type, so a Prometheus scrape (and anything sending `*/*`) got
+  HTTP 400. The header is now parsed as the list of media ranges it is, and OpenMetrics is served
+  when the client asks for it.
+type: fixed
+authors:
+  - name: Jan Høydahl
+    url: https://home.apache.org/phonebook.html?uid=janhoy
+links:
+  - name: GITHUB#4210
+    url: https://github.com/apache/solr/pull/4210

changelog/unreleased/SOLR-18147-grafana-dashboard-solr10.yml

@@ -0,0 +1,8 @@
+title: Add a new Grafana dashboard for Solr 10. This uses the new OTEL based metrics introduced in 10.0 and does not rely on solr-exporter
+type: added
+authors:
+  - name: Jan Høydahl
+    url: https://home.apache.org/phonebook.html?uid=janhoy
+links:
+  - name: SOLR-18147
+    url: https://issues.apache.org/jira/browse/SOLR-18147

solr/core/src/java/org/apache/solr/handler/admin/api/GetMetrics.java

@@ -26,6 +26,7 @@
 import java.lang.invoke.MethodHandles;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Locale;
 import java.util.Map;
 import java.util.Set;
 import java.util.SortedMap;
@@ -122,18 +123,44 @@ private void validateRequest(String acceptHeader) {
     }
 
     // Should handle 'Accept' header only, but a lot of code still expects 'wt'.
-    if (acceptHeader == null) {
-      solrQueryRequest.setParams(
-          SolrParams.wrapDefaults(
-              solrQueryRequest.getParams(),
-              SolrParams.of(CommonParams.WT, MetricUtils.PROMETHEUS_METRICS_WT)));
-    } else if (!PrometheusResponseWriter.CONTENT_TYPE_PROMETHEUS.equals(acceptHeader)
-        && !PrometheusResponseWriter.CONTENT_TYPE_OPEN_METRICS.equals(acceptHeader)) {
+    String wt = metricsFormat(acceptHeader);
+    if (wt == null) {
       throw new SolrException(
           SolrException.ErrorCode.BAD_REQUEST,
           "Only Prometheus and OpenMetrics metric formats supported. Unsupported format requested: "
               + acceptHeader);
     }
+    solrQueryRequest.setParams(
+        SolrParams.wrapDefaults(solrQueryRequest.getParams(), SolrParams.of(CommonParams.WT, wt)));
+  }
+
+  /**
+   * Resolves the {@code wt} to answer an {@code Accept} header with. The header is a comma
+   * separated list of media ranges, each with optional parameters, so it cannot be compared against
+   * a content type directly. Returns null when the client accepts no format this endpoint produces.
+   */
+  static String metricsFormat(String acceptHeader) {
+    if (acceptHeader == null || acceptHeader.isBlank()) {
+      return MetricUtils.PROMETHEUS_METRICS_WT;
+    }
+    boolean prometheusAccepted = false;
+    for (String mediaRange : acceptHeader.split(",")) {
+      String[] parts = mediaRange.trim().split(";");
+      String mediaType = parts[0].trim().toLowerCase(Locale.ROOT);
+      if (PrometheusResponseWriter.MEDIA_TYPE_OPEN_METRICS.equals(mediaType)) {
+        for (int i = 1; i < parts.length; i++) {
+          if (parts[i].replace(" ", "").equalsIgnoreCase("version=1.0.0")) {
+            return MetricUtils.OPEN_METRICS_WT;
+          }
+        }
+        prometheusAccepted = true;
+      } else if (PrometheusResponseWriter.MEDIA_TYPE_PROMETHEUS.equals(mediaType)
+          || "text/*".equals(mediaType)
+          || "*/*".equals(mediaType)) {
+        prometheusAccepted = true;
+      }
+    }
+    return prometheusAccepted ? MetricUtils.PROMETHEUS_METRICS_WT : null;
   }
 
   private boolean proxyToNodes() {

solr/core/src/java/org/apache/solr/response/PrometheusResponseWriter.java

@@ -36,9 +36,12 @@
 public class PrometheusResponseWriter implements QueryResponseWriter {
   // not TextQueryResponseWriter because Prometheus libs work with an OutputStream
 
-  public static final String CONTENT_TYPE_PROMETHEUS = "text/plain; version=0.0.4";
+  public static final String MEDIA_TYPE_PROMETHEUS = "text/plain";
+  public static final String MEDIA_TYPE_OPEN_METRICS = "application/openmetrics-text";
+
+  public static final String CONTENT_TYPE_PROMETHEUS = MEDIA_TYPE_PROMETHEUS + "; version=0.0.4";
   public static final String CONTENT_TYPE_OPEN_METRICS =
-      "application/openmetrics-text; version=1.0.0; charset=utf-8";
+      MEDIA_TYPE_OPEN_METRICS + "; version=1.0.0; charset=utf-8";
 
   @Override
   public void write(

solr/core/src/test/org/apache/solr/handler/admin/api/GetMetricsTest.java

@@ -271,6 +271,75 @@ public void testGetMetricsProxyToNode() throws IOException {
     Assert.assertEquals(200, response.getStatus());
   }
 
+  @Test
+  public void testGetMetricsAcceptHeaderWithMediaRangeList() {
+    // The header a Prometheus scraper actually sends: a list of media ranges with q values.
+    String prometheusScraperAccept =
+        "application/openmetrics-text;version=1.0.0,application/openmetrics-text;version=0.0.1;q=0.75,"
+            + "text/plain;version=1.0.0;q=0.5,text/plain;version=0.0.4;q=0.3,*/*;q=0.1";
+
+    ContentResponse response = requestWithAccept(prometheusScraperAccept);
+    Assert.assertEquals(200, response.getStatus());
+    Assert.assertEquals(PrometheusResponseWriter.MEDIA_TYPE_OPEN_METRICS, response.getMediaType());
+
+    // No space after the semicolon, as emitted by most clients
+    response = requestWithAccept("text/plain;version=0.0.4");
+    Assert.assertEquals(200, response.getStatus());
+    Assert.assertEquals(PrometheusResponseWriter.MEDIA_TYPE_PROMETHEUS, response.getMediaType());
+
+    response = requestWithAccept("*/*");
+    Assert.assertEquals(200, response.getStatus());
+    Assert.assertEquals(PrometheusResponseWriter.MEDIA_TYPE_PROMETHEUS, response.getMediaType());
+  }
+
+  @Test
+  public void testGetMetricsUnsupportedAcceptHeader() {
+    ContentResponse response = requestWithAccept("application/json");
+    Assert.assertEquals(400, response.getStatus());
+  }
+
+  @Test
+  public void testMetricsFormatNegotiation() {
+    Assert.assertEquals(MetricUtils.PROMETHEUS_METRICS_WT, GetMetrics.metricsFormat(null));
+    Assert.assertEquals(MetricUtils.PROMETHEUS_METRICS_WT, GetMetrics.metricsFormat(""));
+    Assert.assertEquals(MetricUtils.PROMETHEUS_METRICS_WT, GetMetrics.metricsFormat("*/*"));
+    Assert.assertEquals(
+        MetricUtils.PROMETHEUS_METRICS_WT,
+        GetMetrics.metricsFormat(PrometheusResponseWriter.CONTENT_TYPE_PROMETHEUS));
+    Assert.assertEquals(
+        MetricUtils.OPEN_METRICS_WT,
+        GetMetrics.metricsFormat(PrometheusResponseWriter.CONTENT_TYPE_OPEN_METRICS));
+    Assert.assertEquals(
+        MetricUtils.OPEN_METRICS_WT,
+        GetMetrics.metricsFormat("application/openmetrics-text;version=1.0.0,text/plain;q=0.3"));
+    // OpenMetrics 0.0.1 is not produced, so the Prometheus text format answers it
+    Assert.assertEquals(
+        MetricUtils.PROMETHEUS_METRICS_WT,
+        GetMetrics.metricsFormat("application/openmetrics-text;version=0.0.1"));
+    Assert.assertNull(GetMetrics.metricsFormat("application/json"));
+  }
+
+  private static ContentResponse requestWithAccept(String acceptHeader) {
+    try {
+      return jettyHttpClient
+          .newRequest(metricsV2Url)
+          .timeout(TIMEOUT, TimeUnit.MILLISECONDS)
+          .method(HttpMethod.GET)
+          .headers(
+              new Consumer<Mutable>() {
+
+                @Override
+                public void accept(Mutable arg0) {
+                  arg0.add(HttpHeader.ACCEPT, acceptHeader);
+                }
+              })
+          .send();
+    } catch (InterruptedException | ExecutionException | TimeoutException e) {
+      Assert.fail("Should not throw exception: " + e.getClass() + ".  message: " + e.getMessage());
+      return null;
+    }
+  }
+
   private static String readMaxOut(byte[] bytes) throws IOException {
     int max = bytes.length > MAX_OUTPUT ? MAX_OUTPUT : bytes.length;
     String str = "";