Parse unavailable indexer health messages and emit metric

internal/arr/collector/health.go

@@ -11,12 +11,18 @@ import (
 )
 
 type systemHealthCollector struct {
-	config             *config.ArrConfig // App configuration
-	systemHealthMetric *prometheus.Desc  // Total number of health issues
-	errorMetric        *prometheus.Desc  // Error Description for use with InvalidMetric
+	config             *config.ArrConfig          // App configuration
+	systemHealthMetric *prometheus.Desc           // Total number of health issues
+	errorMetric        *prometheus.Desc           // Error Description for use with InvalidMetric
+	extraEmitters      []ExtraHealthMetricEmitter // Registered Emitters for extra per-app metrics
 }
 
-func NewSystemHealthCollector(c *config.ArrConfig) *systemHealthCollector {
+type ExtraHealthMetricEmitter interface {
+	Describe() *prometheus.Desc
+	Emit(model.SystemHealthMessage) []prometheus.Metric
+}
+
+func NewSystemHealthCollector(c *config.ArrConfig, emitters ...ExtraHealthMetricEmitter) *systemHealthCollector {
 	return &systemHealthCollector{
 		config: c,
 		systemHealthMetric: prometheus.NewDesc(
@@ -31,11 +37,15 @@ func NewSystemHealthCollector(c *config.ArrConfig) *systemHealthCollector {
 			nil,
 			prometheus.Labels{"url": c.URL},
 		),
+		extraEmitters: emitters,
 	}
 }
 
 func (collector *systemHealthCollector) Describe(ch chan<- *prometheus.Desc) {
 	ch <- collector.systemHealthMetric
+	for _, emitter := range collector.extraEmitters {
+		ch <- emitter.Describe()
+	}
 }
 
 func (collector *systemHealthCollector) Collect(ch chan<- prometheus.Metric) {
@@ -58,6 +68,11 @@ func (collector *systemHealthCollector) Collect(ch chan<- prometheus.Metric) {
 			ch <- prometheus.MustNewConstMetric(collector.systemHealthMetric, prometheus.GaugeValue, float64(1),
 				s.Source, s.Type, s.Message, s.WikiURL,
 			)
+			for _, emitter := range collector.extraEmitters {
+				for _, metric := range emitter.Emit(s) {
+					ch <- metric
+				}
+			}
 		}
 	} else {
 		ch <- prometheus.MustNewConstMetric(collector.systemHealthMetric, prometheus.GaugeValue, float64(0), "", "", "", "")

internal/arr/collector/prowlarr.go

@@ -1,6 +1,7 @@
 package collector
 
 import (
+	"strings"
 	"sync"
 	"time"
 
@@ -91,6 +92,42 @@ func (u *userAgentStatCache) UpdateKey(key string, value model.UserAgentStats) m
 	return entry
 }
 
+type UnavailableIndexerEmitter struct {
+	url string
+}
+
+func NewUnavailableIndexerEmitter(url string) *UnavailableIndexerEmitter {
+	return &UnavailableIndexerEmitter{
+		url: url,
+	}
+}
+
+func (e *UnavailableIndexerEmitter) Describe() *prometheus.Desc {
+	return prometheus.NewDesc(
+		"prowlarr_indexer_unavailable",
+		"Indexers marked unavailable due to repeated errors",
+		[]string{"indexer"},
+		prometheus.Labels{"url": e.url},
+	)
+}
+
+func (e *UnavailableIndexerEmitter) Emit(msg model.SystemHealthMessage) []prometheus.Metric {
+	ret := []prometheus.Metric{}
+	if msg.Source == "IndexerStatusCheck" || msg.Source == "IndexerLongTermStatusCheck" {
+		parts := strings.Split(msg.Message, ":")
+		svrs := parts[len(parts)-1]
+		for _, svr := range strings.Split(svrs, ",") {
+			ret = append(ret, prometheus.MustNewConstMetric(
+				e.Describe(),
+				prometheus.GaugeValue,
+				1,
+				strings.TrimSpace(svr),
+			))
+		}
+	}
+	return ret
+}
+
 type prowlarrCollector struct {
 	config                           *config.ArrConfig  // App configuration
 	indexerStatCache                 indexerStatCache   // Cache of indexer stats

internal/arr/collector/prowlarr_test.go

@@ -0,0 +1,60 @@
+package collector
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/onedr0p/exportarr/internal/arr/model"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/testutil"
+	"github.com/stretchr/testify/require"
+)
+
+type testCollector struct {
+	emitter ExtraHealthMetricEmitter
+	msg     model.SystemHealthMessage
+}
+
+func (c *testCollector) Describe(ch chan<- *prometheus.Desc) {
+	ch <- c.emitter.Describe()
+}
+
+func (c *testCollector) Collect(ch chan<- prometheus.Metric) {
+	for _, metric := range c.emitter.Emit(c.msg) {
+		ch <- metric
+	}
+}
+
+func TestUnavailableIndexerEmitter(t *testing.T) {
+	emitter := UnavailableIndexerEmitter{
+		url: "http://localhost:9117",
+	}
+
+	require := require.New(t)
+	require.NotNil(emitter.Describe())
+
+	msg := model.SystemHealthMessage{
+		Source:  "IndexerLongTermStatusCheck",
+		Type:    "warning",
+		WikiURL: "https://wiki.servarr.com/prowlarr/system#indexers-are-unavailable-due-to-failures",
+		Message: "Indexers unavailable due to failures for more than 6 hours: Server1, ServerTwo, ServerTHREE, Server.four",
+	}
+	metrics := emitter.Emit(msg)
+	require.Len(metrics, 4)
+
+	testCol := &testCollector{
+		emitter: &emitter,
+		msg:     msg,
+	}
+
+	expected := strings.NewReader(
+		`# HELP prowlarr_indexer_unavailable Indexers marked unavailable due to repeated errors
+		# TYPE prowlarr_indexer_unavailable gauge
+		prowlarr_indexer_unavailable{indexer="Server.four",url="http://localhost:9117"} 1
+		prowlarr_indexer_unavailable{indexer="Server1",url="http://localhost:9117"} 1
+		prowlarr_indexer_unavailable{indexer="ServerTHREE",url="http://localhost:9117"} 1
+		prowlarr_indexer_unavailable{indexer="ServerTwo",url="http://localhost:9117"} 1
+		`)
+	err := testutil.CollectAndCompare(testCol, expected)
+	require.NoError(err)
+}

internal/arr/model/shared.go

@@ -42,8 +42,10 @@ type History struct {
 	TotalRecords int `json:"totalRecords"`
 }
 
+type SystemHealth []SystemHealthMessage
+
 // SystemHealth - Stores struct of JSON response
-type SystemHealth []struct {
+type SystemHealthMessage struct {
 	Source  string `json:"source"`
 	Type    string `json:"type"`
 	Message string `json:"message"`

internal/commands/arr.go

@@ -166,7 +166,8 @@ var prowlarrCmd = &cobra.Command{
 				collector.NewProwlarrCollector(c),
 				collector.NewHistoryCollector(c),
 				collector.NewSystemStatusCollector(c),
-				collector.NewSystemHealthCollector(c),
+				collector.NewSystemHealthCollector(c,
+					collector.NewUnavailableIndexerEmitter(c.URL)),
 			)
 		})
 		return nil