Add HTML pages to view tenants with overrides

CHANGELOG.md

@@ -41,7 +41,7 @@
 * [ENHANCEMENT] Added a `parquet-page` cache role for page level caching. [#3196](https://github.com/grafana/tempo/pull/3196) (@joe-elliott)
 * [ENHANCEMENT] Update opentelemetry-collector-contrib dependency to the latest version, v0.89.0 [#3148](https://github.com/grafana/tempo/pull/3148) (@gebn)
 * [ENHANCEMENT] Update memcached default image in jsonnet for multiple CVE [#3310](https://github.com/grafana/tempo/pull/3310) (@zalegrala)
-* [ENHANCEMENT] Add /status/overrides/{tenant} endpoint [#3244](https://github.com/grafana/tempo/pull/3244) (@kvrhdn)
+* [ENHANCEMENT] Add HTML pages /status/overrides and /status/overrides/{tenant} [#3244](https://github.com/grafana/tempo/pull/3244) [#3332](https://github.com/grafana/tempo/pull/3332) (@kvrhdn)
 * [BUGFIX] Prevent building parquet iterators that would loop forever. [#3159](https://github.com/grafana/tempo/pull/3159) (@mapno)
 * [BUGFIX] Sanitize name in mapped dimensions in span-metrics processor [#3171](https://github.com/grafana/tempo/pull/3171) (@mapno)
 * [BUGFIX] Fixed an issue where cached footers were requested then ignored. [#3196](https://github.com/grafana/tempo/pull/3196) (@joe-elliott)

cmd/tempo/app/app.go

@@ -199,7 +199,6 @@ func (t *App) Run() error {
 	t.Server.HTTP().Path("/ready").Handler(t.readyHandler(sm))
 	t.Server.HTTP().Path("/status").Handler(t.statusHandler()).Methods("GET")
 	t.Server.HTTP().Path("/status/{endpoint}").Handler(t.statusHandler()).Methods("GET")
-	t.Server.HTTP().Path("/status/overrides/{tenant}").Handler(t.tenantOverridesHandler()).Methods("GET")
 	grpc_health_v1.RegisterHealthServer(t.Server.GRPC(), grpcutil.NewHealthCheck(sm))
 
 	// Let's listen for events from this manager, and log them.
@@ -373,8 +372,6 @@ func (t *App) statusHandler() http.HandlerFunc {
 
 			switch endpoint {
 			case "runtime_config":
-				fallthrough
-			case "overrides":
 				err := t.writeRuntimeConfig(&msg, r)
 				if err != nil {
 					errs = append(errs, err)
@@ -523,25 +520,6 @@ func (t *App) writeStatusEndpoints(w io.Writer) error {
 	return nil
 }
 
-func (t *App) tenantOverridesHandler() http.HandlerFunc {
-	return func(w http.ResponseWriter, r *http.Request) {
-		vars := mux.Vars(r)
-
-		tenant, ok := vars["tenant"]
-		if !ok {
-			http.Error(w, "must specify tenant ID", http.StatusBadRequest)
-			return
-		}
-
-		err := t.Overrides.WriteTenantOverrides(w, r, tenant)
-		if err != nil {
-			http.Error(w, err.Error(), http.StatusInternalServerError)
-			level.Error(log.Logger).Log("msg", "error writing response", "endpoint", r.URL.String(), "err", err)
-			return
-		}
-	}
-}
-
 func (t *App) buildinfoHandler() http.HandlerFunc {
 	return func(w http.ResponseWriter, r *http.Request) {
 		w.Header().Set("Content-Type", "application/json")

cmd/tempo/app/modules.go

@@ -182,6 +182,9 @@ func (t *App) initOverrides() (services.Service, error) {
 		prometheus.MustRegister(t.Overrides)
 	}
 
+	t.Server.HTTP().Path("/status/overrides").HandlerFunc(overrides.TenantsHandler(t.Overrides)).Methods("GET")
+	t.Server.HTTP().Path("/status/overrides/{tenant}").HandlerFunc(overrides.TenantStatusHandler(t.Overrides)).Methods("GET")
+
 	return t.Overrides, nil
 }
 

docs/sources/tempo/api_docs/_index.md

@@ -560,13 +560,17 @@ Displays the override configuration.
 Query parameter:
 - `mode = (diff)`: Show the difference between defaults and overrides.
 
-Alias: `/status/overrides`
+```
+GET /status/overrides
+```
+
+Displays all tenants that have non-default overrides configured. 
 
 ```
 GET /status/overrides/{tenant}
 ```
 
-Displays the override configuration for the specified tenant.
+Displays all overrides configured for the specified tenant.
 
 ```
 GET /status/usage-stats

modules/overrides/interface.go

@@ -21,6 +21,9 @@ type Service interface {
 type Interface interface {
 	prometheus.Collector
 
+	// GetTenantIDs returns all tenants that have non-default overrides.
+	GetTenantIDs() []string
+
 	// GetRuntimeOverridesFor returns the runtime overrides set for the given user excluding
 	// overrides from the user-configurable overrides, if enabled.
 	GetRuntimeOverridesFor(userID string) *Overrides
@@ -68,5 +71,4 @@ type Interface interface {
 
 	// Management API
 	WriteStatusRuntimeConfig(w io.Writer, r *http.Request) error
-	WriteTenantOverrides(w io.Writer, r *http.Request, tenant string) error
 }

modules/overrides/overrides_tenant_status_http.go

@@ -0,0 +1,88 @@
+package overrides
+
+import (
+	_ "embed" // Used to embed html templates
+	"fmt"
+	"html/template"
+	"net/http"
+	"time"
+
+	"github.com/gorilla/mux"
+	"golang.org/x/exp/slices"
+	"gopkg.in/yaml.v2"
+
+	"github.com/grafana/tempo/pkg/util"
+)
+
+//go:embed tenant_status.gohtml
+var tenantStatusPageHTML string
+var tenantStatusTemplate = template.Must(template.New("webpage").Parse(tenantStatusPageHTML))
+
+type tenantStatusPageContents struct {
+	Now                       time.Time `json:"now"`
+	Tenant                    string    `json:"tenant"`
+	UserConfigurableOverrides string    `json:"user_configurable_overrides"`
+	RuntimeOverrides          string    `json:"runtime_overrides"`
+	RuntimeOverridesSource    string    `json:"using_default_or_wildcard_runtime_overrides"`
+}
+
+func TenantStatusHandler(o Interface) http.HandlerFunc {
+	return func(w http.ResponseWriter, req *http.Request) {
+		page := tenantStatusPageContents{
+			Now: time.Now(),
+		}
+
+		vars := mux.Vars(req)
+
+		page.Tenant = vars["tenant"]
+		if page.Tenant == "" {
+			util.WriteTextResponse(w, "Tenant ID can't be empty")
+			return
+		}
+
+		// runtime overrides
+		overrides := o.GetRuntimeOverridesFor(page.Tenant)
+		runtimeOverrides, err := yaml.Marshal(overrides)
+		if err != nil {
+			util.WriteTextResponse(w, fmt.Sprintf("Marshalling runtime overrides failed: %s", err))
+			return
+		}
+		page.RuntimeOverrides = string(runtimeOverrides)
+
+		var runtimeTenants []string
+		switch o := o.(type) {
+		case *runtimeConfigOverridesManager:
+			runtimeTenants = o.GetTenantIDs()
+		case *userConfigurableOverridesManager:
+			runtimeTenants = o.Interface.GetTenantIDs()
+		default:
+			util.WriteTextResponse(w, "Internal error happened when retrieving runtime overrides")
+		}
+		if slices.Contains(runtimeTenants, page.Tenant) {
+			page.RuntimeOverridesSource = page.Tenant
+		} else if slices.Contains(runtimeTenants, wildcardTenant) {
+			page.RuntimeOverridesSource = wildcardTenant
+		} else {
+			page.RuntimeOverridesSource = "default overrides"
+		}
+
+		// user-configurable overrides
+		if userConfigOverridesManager, ok := o.(*userConfigurableOverridesManager); ok {
+			overrides := userConfigOverridesManager.getTenantLimits(page.Tenant)
+			if overrides != nil {
+				marshalledOverrides, err := yaml.Marshal(overrides)
+				if err != nil {
+					util.WriteTextResponse(w, fmt.Sprintf("Marshalling user-configurable overrides failed: %s", err))
+					return
+				}
+				page.UserConfigurableOverrides = string(marshalledOverrides)
+			} else {
+				page.UserConfigurableOverrides = "No user-configurable overrides set"
+			}
+		} else {
+			page.UserConfigurableOverrides = "User-configurable overrides are not enabled"
+		}
+
+		util.RenderHTTPResponse(w, page, tenantStatusTemplate, req)
+	}
+}

modules/overrides/overrides_tenants_http.go

@@ -0,0 +1,80 @@
+package overrides
+
+import (
+	_ "embed" // Used to embed html templates
+	"html/template"
+	"net/http"
+	"sort"
+	"strings"
+	"time"
+
+	"golang.org/x/exp/maps"
+
+	"github.com/grafana/tempo/pkg/util"
+)
+
+//go:embed tenants.gohtml
+var tenantsPageHTML string
+var tenantsTemplate = template.Must(template.New("webpage").Parse(tenantsPageHTML))
+
+type tenantsPageContents struct {
+	Now     time.Time            `json:"now"`
+	Tenants []*tenantsPageTenant `json:"tenants,omitempty"`
+}
+
+type tenantsPageTenant struct {
+	Name                         string `json:"name"`
+	HasRuntimeOverrides          bool   `json:"has_runtime_overrides"`
+	HasUserConfigurableOverrides bool   `json:"has_user_configurable_overrides"`
+}
+
+func TenantsHandler(o Interface) http.HandlerFunc {
+	return func(w http.ResponseWriter, req *http.Request) {
+		tenants := make(map[string]*tenantsPageTenant)
+
+		// runtime overrides
+		var runtimeTenants []string
+		switch o := o.(type) {
+		case *runtimeConfigOverridesManager:
+			runtimeTenants = o.GetTenantIDs()
+		case *userConfigurableOverridesManager:
+			runtimeTenants = o.Interface.GetTenantIDs()
+		default:
+			util.WriteTextResponse(w, "Internal error happened when retrieving runtime overrides")
+		}
+		for _, tenant := range runtimeTenants {
+			tenants[tenant] = &tenantsPageTenant{
+				Name:                tenant,
+				HasRuntimeOverrides: true,
+			}
+		}
+
+		// user-configurable overrides
+		userConfigurableOverridesManager, ok := o.(*userConfigurableOverridesManager)
+		if ok {
+			for _, tenant := range userConfigurableOverridesManager.GetTenantIDs() {
+				tenantsPage := tenants[tenant]
+				if tenantsPage == nil {
+					tenantsPage = &tenantsPageTenant{Name: tenant}
+					tenants[tenant] = tenantsPage
+				}
+
+				tenantsPage.HasUserConfigurableOverrides = true
+			}
+		}
+
+		tenantsList := maps.Values(tenants)
+		sortTenantsPageTenant(tenantsList)
+
+		util.RenderHTTPResponse(w, tenantsPageContents{
+			Now:     time.Now(),
+			Tenants: tenantsList,
+		}, tenantsTemplate, req)
+	}
+}
+
+func sortTenantsPageTenant(list []*tenantsPageTenant) {
+	sort.Slice(list, func(i, j int) bool {
+		return strings.Compare(list[i].Name, list[j].Name) < 0
+	})
+}

modules/overrides/runtime_config_overrides.go

@@ -8,6 +8,7 @@ import (
 	"time"
 
 	"github.com/go-kit/log/level"
+	"golang.org/x/exp/maps"
 
 	"github.com/grafana/dskit/runtimeconfig"
 	"github.com/grafana/dskit/services"
@@ -241,20 +242,17 @@ func (o *runtimeConfigOverridesManager) WriteStatusRuntimeConfig(w io.Writer, r
 	return nil
 }
 
-func (o *runtimeConfigOverridesManager) GetRuntimeOverridesFor(userID string) *Overrides {
-	return o.getOverridesForUser(userID)
-}
-
-func (o *runtimeConfigOverridesManager) WriteTenantOverrides(w io.Writer, _ *http.Request, userID string) error {
-	overrides := o.getOverridesForUser(userID)
-
-	out, err := yaml.Marshal(overrides)
-	if err != nil {
-		return err
+func (o *runtimeConfigOverridesManager) GetTenantIDs() []string {
+	tenantOverrides := o.tenantOverrides()
+	if tenantOverrides == nil {
+		return nil
 	}
 
-	_, err = w.Write(out)
-	return err
+	return maps.Keys(tenantOverrides.TenantLimits)
+}
+
+func (o *runtimeConfigOverridesManager) GetRuntimeOverridesFor(userID string) *Overrides {
+	return o.getOverridesForUser(userID)
 }
 
 // IngestionRateStrategy returns whether the ingestion rate limit should be individually applied

modules/overrides/tenant_status.gohtml

@@ -0,0 +1,18 @@
+{{- /*gotype: github.com/grafana/tempo/modules/overrides.tenantStatusPageContents*/ -}}
+<!DOCTYPE html>
+<html>
+<head>
+    <meta charset="UTF-8">
+    <title>Overrides: tenant</title>
+</head>
+<body>
+<h1>Overrides: tenant</h1>
+<p>Current time: {{ .Now }}</p>
+<p>Showing overrides for tenant: <strong>{{ .Tenant }}</strong></p>
+<h3>User-configurable overrides</h3>
+<pre><code>{{ .UserConfigurableOverrides }}</code></pre>
+<h3>Runtime overrides</h3>
+<p>Source of runtime overrides: <strong>{{ .RuntimeOverridesSource }}</strong></p>
+<pre><code>{{ .RuntimeOverrides }}</code></pre>
+</body>
+</html>

modules/overrides/tenants.gohtml

@@ -0,0 +1,30 @@
+{{- /*gotype: github.com/grafana/tempo/modules/overrides.tenantsPageContents*/ -}}
+<!DOCTYPE html>
+<html>
+<head>
+    <meta charset="UTF-8">
+    <title>Overrides: tenants</title>
+</head>
+<body>
+<h1>Overrides: tenants</h1>
+<p>Current time: {{ .Now }}</p>
+<table border="1" cellpadding="5" style="border-collapse: collapse">
+    <thead>
+    <tr>
+        <th>Tenant</th>
+        <th>Runtime overrides</th>
+        <th>User-configurable overrides</th>
+    </tr>
+    </thead>
+    <tbody style="font-family: monospace;">
+    {{ range .Tenants }}
+        <tr>
+            <td><a href="overrides/{{ .Name }}">{{ .Name }}</a></td>
+            <td>{{ if .HasRuntimeOverrides }}x{{ end }}</td>
+            <td>{{ if .HasUserConfigurableOverrides }}x{{ end }}</td>
+        </tr>
+    {{ end }}
+    </tbody>
+</table>
+</body>
+</html>