From 08e42be50b9a082e64c68d353b54a01c9165652a Mon Sep 17 00:00:00 2001
From: Koenraad Verheyden <koenraad.verheyden@grafana.com>
Date: Thu, 25 Jan 2024 20:03:21 +0100
Subject: [PATCH 1/8] Add HTML pages to view tenants with overrides

---
 cmd/tempo/app/app.go                          | 22 -----
 cmd/tempo/app/modules.go                      |  3 +
 modules/overrides/interface.go                |  7 +-
 .../overrides/overrides_tenant_status_http.go | 94 +++++++++++++++++++
 modules/overrides/overrides_tenants_http.go   | 87 +++++++++++++++++
 modules/overrides/runtime_config_overrides.go | 23 ++---
 modules/overrides/tenant_status.gohtml        | 17 ++++
 modules/overrides/tenants.gohtml              | 30 ++++++
 .../overrides/user_configurable_overrides.go  | 29 ++----
 9 files changed, 258 insertions(+), 54 deletions(-)
 create mode 100644 modules/overrides/overrides_tenant_status_http.go
 create mode 100644 modules/overrides/overrides_tenants_http.go
 create mode 100644 modules/overrides/tenant_status.gohtml
 create mode 100644 modules/overrides/tenants.gohtml

diff --git a/cmd/tempo/app/app.go b/cmd/tempo/app/app.go
index e7ff35d14df..3a5f3a47305 100644
--- a/cmd/tempo/app/app.go
+++ b/cmd/tempo/app/app.go
@@ -199,7 +199,6 @@ func (t *App) Run() error {
 	t.Server.HTTP().Path("/ready").Handler(t.readyHandler(sm))
 	t.Server.HTTP().Path("/status").Handler(t.statusHandler()).Methods("GET")
 	t.Server.HTTP().Path("/status/{endpoint}").Handler(t.statusHandler()).Methods("GET")
-	t.Server.HTTP().Path("/status/overrides/{tenant}").Handler(t.tenantOverridesHandler()).Methods("GET")
 	grpc_health_v1.RegisterHealthServer(t.Server.GRPC(), grpcutil.NewHealthCheck(sm))
 
 	// Let's listen for events from this manager, and log them.
@@ -373,8 +372,6 @@ func (t *App) statusHandler() http.HandlerFunc {
 
 			switch endpoint {
 			case "runtime_config":
-				fallthrough
-			case "overrides":
 				err := t.writeRuntimeConfig(&msg, r)
 				if err != nil {
 					errs = append(errs, err)
@@ -523,25 +520,6 @@ func (t *App) writeStatusEndpoints(w io.Writer) error {
 	return nil
 }
 
-func (t *App) tenantOverridesHandler() http.HandlerFunc {
-	return func(w http.ResponseWriter, r *http.Request) {
-		vars := mux.Vars(r)
-
-		tenant, ok := vars["tenant"]
-		if !ok {
-			http.Error(w, "must specify tenant ID", http.StatusBadRequest)
-			return
-		}
-
-		err := t.Overrides.WriteTenantOverrides(w, r, tenant)
-		if err != nil {
-			http.Error(w, err.Error(), http.StatusInternalServerError)
-			level.Error(log.Logger).Log("msg", "error writing response", "endpoint", r.URL.String(), "err", err)
-			return
-		}
-	}
-}
-
 func (t *App) buildinfoHandler() http.HandlerFunc {
 	return func(w http.ResponseWriter, r *http.Request) {
 		w.Header().Set("Content-Type", "application/json")
diff --git a/cmd/tempo/app/modules.go b/cmd/tempo/app/modules.go
index d8423fd8f20..4c45034a566 100644
--- a/cmd/tempo/app/modules.go
+++ b/cmd/tempo/app/modules.go
@@ -182,6 +182,9 @@ func (t *App) initOverrides() (services.Service, error) {
 		prometheus.MustRegister(t.Overrides)
 	}
 
+	t.Server.HTTP().Path("/status/overrides").Handler(http.HandlerFunc(t.Overrides.TenantsHandler)).Methods("GET")
+	t.Server.HTTP().Path("/status/overrides/{tenant}").Handler(http.HandlerFunc(t.Overrides.TenantStatusHandler)).Methods("GET")
+
 	return t.Overrides, nil
 }
 
diff --git a/modules/overrides/interface.go b/modules/overrides/interface.go
index 650697e02cb..b71d85e08ff 100644
--- a/modules/overrides/interface.go
+++ b/modules/overrides/interface.go
@@ -21,6 +21,9 @@ type Service interface {
 type Interface interface {
 	prometheus.Collector
 
+	// GetTenantIDs returns all tenants that have non-default overrides.
+	GetTenantIDs() []string
+
 	// GetRuntimeOverridesFor returns the runtime overrides set for the given user excluding
 	// overrides from the user-configurable overrides, if enabled.
 	GetRuntimeOverridesFor(userID string) *Overrides
@@ -68,5 +71,7 @@ type Interface interface {
 
 	// Management API
 	WriteStatusRuntimeConfig(w io.Writer, r *http.Request) error
-	WriteTenantOverrides(w io.Writer, r *http.Request, tenant string) error
+
+	TenantsHandler(w http.ResponseWriter, req *http.Request)
+	TenantStatusHandler(w http.ResponseWriter, req *http.Request)
 }
diff --git a/modules/overrides/overrides_tenant_status_http.go b/modules/overrides/overrides_tenant_status_http.go
new file mode 100644
index 00000000000..522503db925
--- /dev/null
+++ b/modules/overrides/overrides_tenant_status_http.go
@@ -0,0 +1,94 @@
+package overrides
+
+import (
+	_ "embed" // Used to embed html templates
+	"fmt"
+	"html/template"
+	"net/http"
+	"time"
+
+	"github.com/gorilla/mux"
+	"gopkg.in/yaml.v2"
+
+	"github.com/grafana/tempo/pkg/util"
+)
+
+//go:embed tenant_status.gohtml
+var tenantStatusPageHTML string
+var tenantStatusTemplate = template.Must(template.New("webpage").Parse(tenantStatusPageHTML))
+
+type tenantStatusPageContents struct {
+	Now                       time.Time `json:"now"`
+	Tenant                    string    `json:"tenant"`
+	RuntimeOverrides          string    `json:"runtime_overrides"`
+	UserConfigurableOverrides string    `json:"user_configurable_overrides"`
+}
+
+func (o *runtimeConfigOverridesManager) TenantStatusHandler(w http.ResponseWriter, req *http.Request) {
+	vars := mux.Vars(req)
+
+	tenantID := vars["tenant"]
+	if tenantID == "" {
+		util.WriteTextResponse(w, "Tenant ID can't be empty")
+		return
+	}
+
+	runtimeOverrides, err := marshalRuntimeOverrides(o, tenantID)
+	if err != nil {
+		util.WriteTextResponse(w, err.Error())
+		return
+	}
+
+	util.RenderHTTPResponse(w, tenantStatusPageContents{
+		Now:                       time.Now(),
+		Tenant:                    tenantID,
+		RuntimeOverrides:          runtimeOverrides,
+		UserConfigurableOverrides: "User-configurable overrides not enabled",
+	}, tenantStatusTemplate, req)
+}
+
+func (o *userConfigurableOverridesManager) TenantStatusHandler(w http.ResponseWriter, req *http.Request) {
+	vars := mux.Vars(req)
+
+	tenantID := vars["tenant"]
+	if tenantID == "" {
+		util.WriteTextResponse(w, "Tenant ID can't be empty")
+		return
+	}
+
+	runtimeOverrides, err := marshalRuntimeOverrides(o, tenantID)
+	if err != nil {
+		util.WriteTextResponse(w, err.Error())
+		return
+	}
+
+	var userConfigurableOverrides string
+
+	overrides := o.getTenantLimits(tenantID)
+	if overrides != nil {
+		marshalledOverrides, err := yaml.Marshal(overrides)
+		if err != nil {
+			util.WriteTextResponse(w, fmt.Sprintf("Marshalling runtime overrides failed: %w", err))
+		}
+		userConfigurableOverrides = string(marshalledOverrides)
+	} else {
+		userConfigurableOverrides = "No user-configurable overrides set"
+	}
+
+	util.RenderHTTPResponse(w, tenantStatusPageContents{
+		Now:                       time.Now(),
+		Tenant:                    tenantID,
+		RuntimeOverrides:          runtimeOverrides,
+		UserConfigurableOverrides: userConfigurableOverrides,
+	}, tenantStatusTemplate, req)
+}
+
+func marshalRuntimeOverrides(o Interface, tenantID string) (string, error) {
+	overrides := o.GetRuntimeOverridesFor(tenantID)
+
+	runtimeOverrides, err := yaml.Marshal(overrides)
+	if err != nil {
+		return "", fmt.Errorf("Marshalling runtime overrides failed: %w", err)
+	}
+	return string(runtimeOverrides), nil
+}
diff --git a/modules/overrides/overrides_tenants_http.go b/modules/overrides/overrides_tenants_http.go
new file mode 100644
index 00000000000..6e712c1319c
--- /dev/null
+++ b/modules/overrides/overrides_tenants_http.go
@@ -0,0 +1,87 @@
+package overrides
+
+import (
+	_ "embed" // Used to embed html templates
+	"html/template"
+	"net/http"
+	"sort"
+	"strings"
+	"time"
+
+	"github.com/grafana/tempo/pkg/util"
+)
+
+//go:embed tenants.gohtml
+var tenantsPageHTML string
+var tenantsTemplate = template.Must(template.New("webpage").Parse(tenantsPageHTML))
+
+type tenantsPageContents struct {
+	Now     time.Time           `json:"now"`
+	Tenants []tenantsPageTenant `json:"tenants,omitempty"`
+}
+
+type tenantsPageTenant struct {
+	Name                         string `json:"name"`
+	HasRuntimeOverrides          bool   `json:"has_runtime_overrides"`
+	HasUserConfigurableOverrides bool   `json:"has_user_configurable_overrides"`
+}
+
+func (o *runtimeConfigOverridesManager) TenantsHandler(w http.ResponseWriter, req *http.Request) {
+	var tenants []tenantsPageTenant
+	for _, tenant := range o.GetTenantIDs() {
+		tenants = append(tenants, tenantsPageTenant{
+			Name:                         tenant,
+			HasRuntimeOverrides:          true,
+			HasUserConfigurableOverrides: false,
+		})
+	}
+
+	sortTenantsPageTenant(tenants)
+
+	util.RenderHTTPResponse(w, tenantsPageContents{
+		Now:     time.Now(),
+		Tenants: tenants,
+	}, tenantsTemplate, req)
+}
+
+func (o *userConfigurableOverridesManager) TenantsHandler(w http.ResponseWriter, req *http.Request) {
+	tenants := make(map[string]tenantsPageTenant)
+
+	// runtime overrides
+	for _, tenant := range o.Interface.GetTenantIDs() {
+		tenants[tenant] = tenantsPageTenant{
+			Name:                         tenant,
+			HasRuntimeOverrides:          true,
+			HasUserConfigurableOverrides: false,
+		}
+	}
+
+	// user-configurable overrides
+	for _, tenant := range o.GetTenantIDs() {
+		_, hasRuntimeOverrides := tenants[tenant]
+
+		tenants[tenant] = tenantsPageTenant{
+			Name:                         tenant,
+			HasRuntimeOverrides:          hasRuntimeOverrides,
+			HasUserConfigurableOverrides: true,
+		}
+	}
+
+	var tenantsList []tenantsPageTenant
+	for _, tenant := range tenants {
+		tenantsList = append(tenantsList, tenant)
+	}
+
+	sortTenantsPageTenant(tenantsList)
+
+	util.RenderHTTPResponse(w, tenantsPageContents{
+		Now:     time.Now(),
+		Tenants: tenantsList,
+	}, tenantsTemplate, req)
+}
+
+func sortTenantsPageTenant(list []tenantsPageTenant) {
+	sort.Slice(list, func(i, j int) bool {
+		return strings.Compare(list[i].Name, list[j].Name) < 0
+	})
+}
diff --git a/modules/overrides/runtime_config_overrides.go b/modules/overrides/runtime_config_overrides.go
index 9708abbab28..d99cd421bfe 100644
--- a/modules/overrides/runtime_config_overrides.go
+++ b/modules/overrides/runtime_config_overrides.go
@@ -241,20 +241,21 @@ func (o *runtimeConfigOverridesManager) WriteStatusRuntimeConfig(w io.Writer, r
 	return nil
 }
 
-func (o *runtimeConfigOverridesManager) GetRuntimeOverridesFor(userID string) *Overrides {
-	return o.getOverridesForUser(userID)
-}
-
-func (o *runtimeConfigOverridesManager) WriteTenantOverrides(w io.Writer, _ *http.Request, userID string) error {
-	overrides := o.getOverridesForUser(userID)
+func (o *runtimeConfigOverridesManager) GetTenantIDs() []string {
+	tenantOverrides := o.tenantOverrides()
+	if tenantOverrides == nil {
+		return nil
+	}
 
-	out, err := yaml.Marshal(overrides)
-	if err != nil {
-		return err
+	var ids []string
+	for tenant := range tenantOverrides.TenantLimits {
+		ids = append(ids, tenant)
 	}
+	return ids
+}
 
-	_, err = w.Write(out)
-	return err
+func (o *runtimeConfigOverridesManager) GetRuntimeOverridesFor(userID string) *Overrides {
+	return o.getOverridesForUser(userID)
 }
 
 // IngestionRateStrategy returns whether the ingestion rate limit should be individually applied
diff --git a/modules/overrides/tenant_status.gohtml b/modules/overrides/tenant_status.gohtml
new file mode 100644
index 00000000000..60b37e21c4a
--- /dev/null
+++ b/modules/overrides/tenant_status.gohtml
@@ -0,0 +1,17 @@
+{{- /*gotype: github.com/grafana/mimir/pkg/storegateway.tenantsPageContents*/ -}}
+<!DOCTYPE html>
+<html>
+<head>
+    <meta charset="UTF-8">
+    <title>Overrides: tenant</title>
+</head>
+<body>
+<h1>Overrides: tenant</h1>
+<p>Current time: {{ .Now }}</p>
+<p>Showing overrides for tenant: <strong>{{ .Tenant }}</strong></p>
+<h3>User-configurable overrides</h3>
+<pre><code>{{ .UserConfigurableOverrides }}</code></pre>
+<h3>Runtime overrides</h3>
+<pre><code>{{ .RuntimeOverrides }}</code></pre>
+</body>
+</html>
diff --git a/modules/overrides/tenants.gohtml b/modules/overrides/tenants.gohtml
new file mode 100644
index 00000000000..53424864f6a
--- /dev/null
+++ b/modules/overrides/tenants.gohtml
@@ -0,0 +1,30 @@
+{{- /*gotype: github.com/grafana/mimir/pkg/storegateway.tenantsPageContents*/ -}}
+<!DOCTYPE html>
+<html>
+<head>
+    <meta charset="UTF-8">
+    <title>Overrides: tenants</title>
+</head>
+<body>
+<h1>Overrides: tenants</h1>
+<p>Current time: {{ .Now }}</p>
+<table border="1" cellpadding="5" style="border-collapse: collapse">
+    <thead>
+    <tr>
+        <th>Tenant</th>
+        <th>Runtime overrides</th>
+        <th>User-configurable overrides</th>
+    </tr>
+    </thead>
+    <tbody style="font-family: monospace;">
+    {{ range .Tenants }}
+        <tr>
+            <td><a href="overrides/{{ .Name }}">{{ .Name }}</a></td>
+            <td>{{ if .HasRuntimeOverrides }}x{{ end }}</td>
+            <td>{{ if .HasUserConfigurableOverrides }}x{{ end }}</td>
+        </tr>
+    {{ end }}
+    </tbody>
+</table>
+</body>
+</html>
diff --git a/modules/overrides/user_configurable_overrides.go b/modules/overrides/user_configurable_overrides.go
index 6a9573355a1..0e1703f7f82 100644
--- a/modules/overrides/user_configurable_overrides.go
+++ b/modules/overrides/user_configurable_overrides.go
@@ -70,6 +70,7 @@ type userConfigurableOverridesManager struct {
 }
 
 var _ Service = (*userConfigurableOverridesManager)(nil)
+var _ Interface = (*userConfigurableOverridesManager)(nil)
 
 // newUserConfigOverrides wraps the given overrides with user-configurable overrides.
 func newUserConfigOverrides(cfg *UserConfigurableOverridesConfig, subOverrides Service) (*userConfigurableOverridesManager, error) {
@@ -195,6 +196,14 @@ func (o *userConfigurableOverridesManager) setTenantLimit(userID string, limits
 	}
 }
 
+func (o *userConfigurableOverridesManager) GetTenantIDs() []string {
+	var ids []string
+	for tenant := range o.getAllTenantLimits() {
+		ids = append(ids, tenant)
+	}
+	return ids
+}
+
 func (o *userConfigurableOverridesManager) Forwarders(userID string) []string {
 	if forwarders, ok := o.getTenantLimits(userID).GetForwarders(); ok {
 		return forwarders
@@ -316,26 +325,6 @@ func (o *userConfigurableOverridesManager) WriteStatusRuntimeConfig(w io.Writer,
 	return nil
 }
 
-type statusTenantOverrides struct {
-	UserConfigurableLimits *userconfigurableoverrides.Limits `yaml:"user_configurable_limits"`
-	RuntimeOverrides       *Overrides                        `yaml:"runtime_overrides"`
-}
-
-func (o *userConfigurableOverridesManager) WriteTenantOverrides(w io.Writer, _ *http.Request, userID string) error {
-	overrides := statusTenantOverrides{
-		UserConfigurableLimits: o.getTenantLimits(userID),
-		RuntimeOverrides:       o.GetRuntimeOverridesFor(userID),
-	}
-
-	out, err := yaml.Marshal(overrides)
-	if err != nil {
-		return err
-	}
-
-	_, err = w.Write(out)
-	return err
-}
-
 func (o *userConfigurableOverridesManager) Describe(ch chan<- *prometheus.Desc) {
 	// TODO for now just pass along to the underlying overrides, in the future we should export
 	//  the user-config overrides as well

From 2b8f49d8765ebc6b2b3af06e8f3b7aa8eb282b7b Mon Sep 17 00:00:00 2001
From: Koenraad Verheyden <koenraad.verheyden@grafana.com>
Date: Thu, 25 Jan 2024 20:15:55 +0100
Subject: [PATCH 2/8] Update CHANGELOG.md

---
 CHANGELOG.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index fb6543358d3..d521aaa3900 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -41,7 +41,7 @@
 * [ENHANCEMENT] Added a `parquet-page` cache role for page level caching. [#3196](https://github.com/grafana/tempo/pull/3196) (@joe-elliott)
 * [ENHANCEMENT] Update opentelemetry-collector-contrib dependency to the latest version, v0.89.0 [#3148](https://github.com/grafana/tempo/pull/3148) (@gebn)
 * [ENHANCEMENT] Update memcached default image in jsonnet for multiple CVE [#3310](https://github.com/grafana/tempo/pull/3310) (@zalegrala)
-* [ENHANCEMENT] Add /status/overrides/{tenant} endpoint [#3244](https://github.com/grafana/tempo/pull/3244) (@kvrhdn)
+* [ENHANCEMENT] Add HTML pages /status/overrides and /status/overrides/{tenant} [#3244](https://github.com/grafana/tempo/pull/3244) [#3332](https://github.com/grafana/tempo/pull/3332) (@kvrhdn)
 * [BUGFIX] Prevent building parquet iterators that would loop forever. [#3159](https://github.com/grafana/tempo/pull/3159) (@mapno)
 * [BUGFIX] Sanitize name in mapped dimensions in span-metrics processor [#3171](https://github.com/grafana/tempo/pull/3171) (@mapno)
 * [BUGFIX] Fixed an issue where cached footers were requested then ignored. [#3196](https://github.com/grafana/tempo/pull/3196) (@joe-elliott)

From f9d19bc3cf00fab7a364e7af8acd452f7203f506 Mon Sep 17 00:00:00 2001
From: Koenraad Verheyden <koenraad.verheyden@grafana.com>
Date: Thu, 25 Jan 2024 20:34:31 +0100
Subject: [PATCH 3/8] Update docs

---
 docs/sources/tempo/api_docs/_index.md | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/docs/sources/tempo/api_docs/_index.md b/docs/sources/tempo/api_docs/_index.md
index 40f9b62368d..a72e20663c3 100644
--- a/docs/sources/tempo/api_docs/_index.md
+++ b/docs/sources/tempo/api_docs/_index.md
@@ -560,13 +560,17 @@ Displays the override configuration.
 Query parameter:
 - `mode = (diff)`: Show the difference between defaults and overrides.
 
-Alias: `/status/overrides`
+```
+GET /status/overrides
+```
+
+Displays all tenants that have non-default overrides configured. 
 
 ```
 GET /status/overrides/{tenant}
 ```
 
-Displays the override configuration for the specified tenant.
+Displays all overrides configured for the specified tenant.
 
 ```
 GET /status/usage-stats

From d3b55ae1583bcc3d50f2e8f2cdb60012f843fa33 Mon Sep 17 00:00:00 2001
From: Koenraad Verheyden <koenraad.verheyden@grafana.com>
Date: Thu, 25 Jan 2024 20:38:40 +0100
Subject: [PATCH 4/8] Linting and fmt

---
 modules/overrides/overrides_tenant_status_http.go | 2 +-
 modules/overrides/user_configurable_overrides.go  | 6 ++++--
 2 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/modules/overrides/overrides_tenant_status_http.go b/modules/overrides/overrides_tenant_status_http.go
index 522503db925..628ce367bc1 100644
--- a/modules/overrides/overrides_tenant_status_http.go
+++ b/modules/overrides/overrides_tenant_status_http.go
@@ -68,7 +68,7 @@ func (o *userConfigurableOverridesManager) TenantStatusHandler(w http.ResponseWr
 	if overrides != nil {
 		marshalledOverrides, err := yaml.Marshal(overrides)
 		if err != nil {
-			util.WriteTextResponse(w, fmt.Sprintf("Marshalling runtime overrides failed: %w", err))
+			util.WriteTextResponse(w, fmt.Sprintf("Marshalling runtime overrides failed: %s", err))
 		}
 		userConfigurableOverrides = string(marshalledOverrides)
 	} else {
diff --git a/modules/overrides/user_configurable_overrides.go b/modules/overrides/user_configurable_overrides.go
index 0e1703f7f82..e77075eed8d 100644
--- a/modules/overrides/user_configurable_overrides.go
+++ b/modules/overrides/user_configurable_overrides.go
@@ -69,8 +69,10 @@ type userConfigurableOverridesManager struct {
 	logger log.Logger
 }
 
-var _ Service = (*userConfigurableOverridesManager)(nil)
-var _ Interface = (*userConfigurableOverridesManager)(nil)
+var (
+	_ Service   = (*userConfigurableOverridesManager)(nil)
+	_ Interface = (*userConfigurableOverridesManager)(nil)
+)
 
 // newUserConfigOverrides wraps the given overrides with user-configurable overrides.
 func newUserConfigOverrides(cfg *UserConfigurableOverridesConfig, subOverrides Service) (*userConfigurableOverridesManager, error) {

From 98db99d4d462d532eb252227ce406125cd1d1f2a Mon Sep 17 00:00:00 2001
From: Koenraad Verheyden <koenraad.verheyden@grafana.com>
Date: Thu, 25 Jan 2024 20:44:51 +0100
Subject: [PATCH 5/8] Clean

---
 cmd/tempo/app/modules.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/cmd/tempo/app/modules.go b/cmd/tempo/app/modules.go
index 4c45034a566..f0c31e0bdc4 100644
--- a/cmd/tempo/app/modules.go
+++ b/cmd/tempo/app/modules.go
@@ -182,8 +182,8 @@ func (t *App) initOverrides() (services.Service, error) {
 		prometheus.MustRegister(t.Overrides)
 	}
 
-	t.Server.HTTP().Path("/status/overrides").Handler(http.HandlerFunc(t.Overrides.TenantsHandler)).Methods("GET")
-	t.Server.HTTP().Path("/status/overrides/{tenant}").Handler(http.HandlerFunc(t.Overrides.TenantStatusHandler)).Methods("GET")
+	t.Server.HTTP().Path("/status/overrides").HandlerFunc(t.Overrides.TenantsHandler).Methods("GET")
+	t.Server.HTTP().Path("/status/overrides/{tenant}").HandlerFunc(t.Overrides.TenantStatusHandler).Methods("GET")
 
 	return t.Overrides, nil
 }

From be204794907a85a330386638789b4903f1f675a6 Mon Sep 17 00:00:00 2001
From: Koenraad Verheyden <koenraad.verheyden@grafana.com>
Date: Tue, 30 Jan 2024 17:14:13 +0100
Subject: [PATCH 6/8] Simplify (?) HTTP handlers, remove handlers from
 Interface

---
 cmd/tempo/app/modules.go                      |  4 +-
 modules/overrides/interface.go                |  3 -
 .../overrides/overrides_tenant_status_http.go | 97 +++++++------------
 modules/overrides/overrides_tenants_http.go   | 88 ++++++++---------
 4 files changed, 81 insertions(+), 111 deletions(-)

diff --git a/cmd/tempo/app/modules.go b/cmd/tempo/app/modules.go
index f0c31e0bdc4..2e5f47a90ec 100644
--- a/cmd/tempo/app/modules.go
+++ b/cmd/tempo/app/modules.go
@@ -182,8 +182,8 @@ func (t *App) initOverrides() (services.Service, error) {
 		prometheus.MustRegister(t.Overrides)
 	}
 
-	t.Server.HTTP().Path("/status/overrides").HandlerFunc(t.Overrides.TenantsHandler).Methods("GET")
-	t.Server.HTTP().Path("/status/overrides/{tenant}").HandlerFunc(t.Overrides.TenantStatusHandler).Methods("GET")
+	t.Server.HTTP().Path("/status/overrides").HandlerFunc(overrides.TenantsHandler(t.Overrides)).Methods("GET")
+	t.Server.HTTP().Path("/status/overrides/{tenant}").HandlerFunc(overrides.TenantStatusHandler(t.Overrides)).Methods("GET")
 
 	return t.Overrides, nil
 }
diff --git a/modules/overrides/interface.go b/modules/overrides/interface.go
index b71d85e08ff..9d5742ff796 100644
--- a/modules/overrides/interface.go
+++ b/modules/overrides/interface.go
@@ -71,7 +71,4 @@ type Interface interface {
 
 	// Management API
 	WriteStatusRuntimeConfig(w io.Writer, r *http.Request) error
-
-	TenantsHandler(w http.ResponseWriter, req *http.Request)
-	TenantStatusHandler(w http.ResponseWriter, req *http.Request)
 }
diff --git a/modules/overrides/overrides_tenant_status_http.go b/modules/overrides/overrides_tenant_status_http.go
index 628ce367bc1..279df0ecf9b 100644
--- a/modules/overrides/overrides_tenant_status_http.go
+++ b/modules/overrides/overrides_tenant_status_http.go
@@ -24,71 +24,48 @@ type tenantStatusPageContents struct {
 	UserConfigurableOverrides string    `json:"user_configurable_overrides"`
 }
 
-func (o *runtimeConfigOverridesManager) TenantStatusHandler(w http.ResponseWriter, req *http.Request) {
-	vars := mux.Vars(req)
-
-	tenantID := vars["tenant"]
-	if tenantID == "" {
-		util.WriteTextResponse(w, "Tenant ID can't be empty")
-		return
-	}
-
-	runtimeOverrides, err := marshalRuntimeOverrides(o, tenantID)
-	if err != nil {
-		util.WriteTextResponse(w, err.Error())
-		return
-	}
-
-	util.RenderHTTPResponse(w, tenantStatusPageContents{
-		Now:                       time.Now(),
-		Tenant:                    tenantID,
-		RuntimeOverrides:          runtimeOverrides,
-		UserConfigurableOverrides: "User-configurable overrides not enabled",
-	}, tenantStatusTemplate, req)
-}
-
-func (o *userConfigurableOverridesManager) TenantStatusHandler(w http.ResponseWriter, req *http.Request) {
-	vars := mux.Vars(req)
-
-	tenantID := vars["tenant"]
-	if tenantID == "" {
-		util.WriteTextResponse(w, "Tenant ID can't be empty")
-		return
-	}
-
-	runtimeOverrides, err := marshalRuntimeOverrides(o, tenantID)
-	if err != nil {
-		util.WriteTextResponse(w, err.Error())
-		return
-	}
-
-	var userConfigurableOverrides string
+func TenantStatusHandler(o Interface) http.HandlerFunc {
+	return func(w http.ResponseWriter, req *http.Request) {
+		vars := mux.Vars(req)
+
+		tenantID := vars["tenant"]
+		if tenantID == "" {
+			util.WriteTextResponse(w, "Tenant ID can't be empty")
+			return
+		}
 
-	overrides := o.getTenantLimits(tenantID)
-	if overrides != nil {
-		marshalledOverrides, err := yaml.Marshal(overrides)
+		// runtime overrides
+		overrides := o.GetRuntimeOverridesFor(tenantID)
+		runtimeOverrides, err := yaml.Marshal(overrides)
 		if err != nil {
 			util.WriteTextResponse(w, fmt.Sprintf("Marshalling runtime overrides failed: %s", err))
+			return
 		}
-		userConfigurableOverrides = string(marshalledOverrides)
-	} else {
-		userConfigurableOverrides = "No user-configurable overrides set"
-	}
-
-	util.RenderHTTPResponse(w, tenantStatusPageContents{
-		Now:                       time.Now(),
-		Tenant:                    tenantID,
-		RuntimeOverrides:          runtimeOverrides,
-		UserConfigurableOverrides: userConfigurableOverrides,
-	}, tenantStatusTemplate, req)
-}
 
-func marshalRuntimeOverrides(o Interface, tenantID string) (string, error) {
-	overrides := o.GetRuntimeOverridesFor(tenantID)
+		// user-configurable overrides
+		var userConfigurableOverrides string
+
+		if userConfigOverridesManager, ok := o.(*userConfigurableOverridesManager); ok {
+			overrides := userConfigOverridesManager.getTenantLimits(tenantID)
+			if overrides != nil {
+				marshalledOverrides, err := yaml.Marshal(overrides)
+				if err != nil {
+					util.WriteTextResponse(w, fmt.Sprintf("Marshalling user-configurable overrides failed: %s", err))
+					return
+				}
+				userConfigurableOverrides = string(marshalledOverrides)
+			} else {
+				userConfigurableOverrides = "No user-configurable overrides set"
+			}
+		} else {
+			userConfigurableOverrides = "User-configurable overrides are not enabled"
+		}
 
-	runtimeOverrides, err := yaml.Marshal(overrides)
-	if err != nil {
-		return "", fmt.Errorf("Marshalling runtime overrides failed: %w", err)
+		util.RenderHTTPResponse(w, tenantStatusPageContents{
+			Now:                       time.Now(),
+			Tenant:                    tenantID,
+			RuntimeOverrides:          string(runtimeOverrides),
+			UserConfigurableOverrides: userConfigurableOverrides,
+		}, tenantStatusTemplate, req)
 	}
-	return string(runtimeOverrides), nil
 }
diff --git a/modules/overrides/overrides_tenants_http.go b/modules/overrides/overrides_tenants_http.go
index 6e712c1319c..1a6685f45fa 100644
--- a/modules/overrides/overrides_tenants_http.go
+++ b/modules/overrides/overrides_tenants_http.go
@@ -26,58 +26,54 @@ type tenantsPageTenant struct {
 	HasUserConfigurableOverrides bool   `json:"has_user_configurable_overrides"`
 }
 
-func (o *runtimeConfigOverridesManager) TenantsHandler(w http.ResponseWriter, req *http.Request) {
-	var tenants []tenantsPageTenant
-	for _, tenant := range o.GetTenantIDs() {
-		tenants = append(tenants, tenantsPageTenant{
-			Name:                         tenant,
-			HasRuntimeOverrides:          true,
-			HasUserConfigurableOverrides: false,
-		})
-	}
-
-	sortTenantsPageTenant(tenants)
-
-	util.RenderHTTPResponse(w, tenantsPageContents{
-		Now:     time.Now(),
-		Tenants: tenants,
-	}, tenantsTemplate, req)
-}
-
-func (o *userConfigurableOverridesManager) TenantsHandler(w http.ResponseWriter, req *http.Request) {
-	tenants := make(map[string]tenantsPageTenant)
-
-	// runtime overrides
-	for _, tenant := range o.Interface.GetTenantIDs() {
-		tenants[tenant] = tenantsPageTenant{
-			Name:                         tenant,
-			HasRuntimeOverrides:          true,
-			HasUserConfigurableOverrides: false,
+func TenantsHandler(o Interface) http.HandlerFunc {
+	return func(w http.ResponseWriter, req *http.Request) {
+		tenants := make(map[string]tenantsPageTenant)
+
+		// runtime overrides
+		var runtimeTenants []string
+		switch o := o.(type) {
+		case *runtimeConfigOverridesManager:
+			runtimeTenants = o.GetTenantIDs()
+		case *userConfigurableOverridesManager:
+			runtimeTenants = o.Interface.GetTenantIDs()
+		default:
+			util.WriteTextResponse(w, "Internal error happened when retrieving runtime overrides")
+		}
+		for _, tenant := range runtimeTenants {
+			tenants[tenant] = tenantsPageTenant{
+				Name:                         tenant,
+				HasRuntimeOverrides:          true,
+				HasUserConfigurableOverrides: false,
+			}
 		}
-	}
-
-	// user-configurable overrides
-	for _, tenant := range o.GetTenantIDs() {
-		_, hasRuntimeOverrides := tenants[tenant]
 
-		tenants[tenant] = tenantsPageTenant{
-			Name:                         tenant,
-			HasRuntimeOverrides:          hasRuntimeOverrides,
-			HasUserConfigurableOverrides: true,
+		// user-configurable overrides
+		userConfigurableOverridesManager, ok := o.(*userConfigurableOverridesManager)
+		if ok {
+			for _, tenant := range userConfigurableOverridesManager.GetTenantIDs() {
+				_, hasRuntimeOverrides := tenants[tenant]
+
+				tenants[tenant] = tenantsPageTenant{
+					Name:                         tenant,
+					HasRuntimeOverrides:          hasRuntimeOverrides,
+					HasUserConfigurableOverrides: true,
+				}
+			}
 		}
-	}
 
-	var tenantsList []tenantsPageTenant
-	for _, tenant := range tenants {
-		tenantsList = append(tenantsList, tenant)
-	}
+		var tenantsList []tenantsPageTenant
+		for _, tenant := range tenants {
+			tenantsList = append(tenantsList, tenant)
+		}
 
-	sortTenantsPageTenant(tenantsList)
+		sortTenantsPageTenant(tenantsList)
 
-	util.RenderHTTPResponse(w, tenantsPageContents{
-		Now:     time.Now(),
-		Tenants: tenantsList,
-	}, tenantsTemplate, req)
+		util.RenderHTTPResponse(w, tenantsPageContents{
+			Now:     time.Now(),
+			Tenants: tenantsList,
+		}, tenantsTemplate, req)
+	}
 }
 
 func sortTenantsPageTenant(list []tenantsPageTenant) {

From 23f68e1b0fb64683f63499a05d0ef3c2b286ceff Mon Sep 17 00:00:00 2001
From: Koenraad Verheyden <koenraad.verheyden@grafana.com>
Date: Thu, 1 Feb 2024 15:26:19 +0100
Subject: [PATCH 7/8] Address review comments

---
 modules/overrides/overrides_tenants_http.go   | 35 +++++++++----------
 modules/overrides/runtime_config_overrides.go |  7 ++--
 modules/overrides/tenant_status.gohtml        |  2 +-
 modules/overrides/tenants.gohtml              |  2 +-
 .../overrides/user_configurable_overrides.go  |  7 ++--
 5 files changed, 22 insertions(+), 31 deletions(-)

diff --git a/modules/overrides/overrides_tenants_http.go b/modules/overrides/overrides_tenants_http.go
index 1a6685f45fa..43b1fb6d99b 100644
--- a/modules/overrides/overrides_tenants_http.go
+++ b/modules/overrides/overrides_tenants_http.go
@@ -8,6 +8,8 @@ import (
 	"strings"
 	"time"
 
+	"golang.org/x/exp/maps"
+
 	"github.com/grafana/tempo/pkg/util"
 )
 
@@ -16,8 +18,8 @@ var tenantsPageHTML string
 var tenantsTemplate = template.Must(template.New("webpage").Parse(tenantsPageHTML))
 
 type tenantsPageContents struct {
-	Now     time.Time           `json:"now"`
-	Tenants []tenantsPageTenant `json:"tenants,omitempty"`
+	Now     time.Time            `json:"now"`
+	Tenants []*tenantsPageTenant `json:"tenants,omitempty"`
 }
 
 type tenantsPageTenant struct {
@@ -28,7 +30,7 @@ type tenantsPageTenant struct {
 
 func TenantsHandler(o Interface) http.HandlerFunc {
 	return func(w http.ResponseWriter, req *http.Request) {
-		tenants := make(map[string]tenantsPageTenant)
+		tenants := make(map[string]*tenantsPageTenant)
 
 		// runtime overrides
 		var runtimeTenants []string
@@ -41,10 +43,9 @@ func TenantsHandler(o Interface) http.HandlerFunc {
 			util.WriteTextResponse(w, "Internal error happened when retrieving runtime overrides")
 		}
 		for _, tenant := range runtimeTenants {
-			tenants[tenant] = tenantsPageTenant{
-				Name:                         tenant,
-				HasRuntimeOverrides:          true,
-				HasUserConfigurableOverrides: false,
+			tenants[tenant] = &tenantsPageTenant{
+				Name:                tenant,
+				HasRuntimeOverrides: true,
 			}
 		}
 
@@ -52,21 +53,17 @@ func TenantsHandler(o Interface) http.HandlerFunc {
 		userConfigurableOverridesManager, ok := o.(*userConfigurableOverridesManager)
 		if ok {
 			for _, tenant := range userConfigurableOverridesManager.GetTenantIDs() {
-				_, hasRuntimeOverrides := tenants[tenant]
-
-				tenants[tenant] = tenantsPageTenant{
-					Name:                         tenant,
-					HasRuntimeOverrides:          hasRuntimeOverrides,
-					HasUserConfigurableOverrides: true,
+				tenantsPage := tenants[tenant]
+				if tenantsPage == nil {
+					tenantsPage = &tenantsPageTenant{Name: tenant}
+					tenants[tenant] = tenantsPage
 				}
-			}
-		}
 
-		var tenantsList []tenantsPageTenant
-		for _, tenant := range tenants {
-			tenantsList = append(tenantsList, tenant)
+				tenantsPage.HasUserConfigurableOverrides = true
+			}
 		}
 
+		tenantsList := maps.Values(tenants)
 		sortTenantsPageTenant(tenantsList)
 
 		util.RenderHTTPResponse(w, tenantsPageContents{
@@ -76,7 +73,7 @@ func TenantsHandler(o Interface) http.HandlerFunc {
 	}
 }
 
-func sortTenantsPageTenant(list []tenantsPageTenant) {
+func sortTenantsPageTenant(list []*tenantsPageTenant) {
 	sort.Slice(list, func(i, j int) bool {
 		return strings.Compare(list[i].Name, list[j].Name) < 0
 	})
diff --git a/modules/overrides/runtime_config_overrides.go b/modules/overrides/runtime_config_overrides.go
index d99cd421bfe..1c6ce6a36c8 100644
--- a/modules/overrides/runtime_config_overrides.go
+++ b/modules/overrides/runtime_config_overrides.go
@@ -8,6 +8,7 @@ import (
 	"time"
 
 	"github.com/go-kit/log/level"
+	"golang.org/x/exp/maps"
 
 	"github.com/grafana/dskit/runtimeconfig"
 	"github.com/grafana/dskit/services"
@@ -247,11 +248,7 @@ func (o *runtimeConfigOverridesManager) GetTenantIDs() []string {
 		return nil
 	}
 
-	var ids []string
-	for tenant := range tenantOverrides.TenantLimits {
-		ids = append(ids, tenant)
-	}
-	return ids
+	return maps.Keys(tenantOverrides.TenantLimits)
 }
 
 func (o *runtimeConfigOverridesManager) GetRuntimeOverridesFor(userID string) *Overrides {
diff --git a/modules/overrides/tenant_status.gohtml b/modules/overrides/tenant_status.gohtml
index 60b37e21c4a..2f46da5cae6 100644
--- a/modules/overrides/tenant_status.gohtml
+++ b/modules/overrides/tenant_status.gohtml
@@ -1,4 +1,4 @@
-{{- /*gotype: github.com/grafana/mimir/pkg/storegateway.tenantsPageContents*/ -}}
+{{- /*gotype: github.com/grafana/tempo/modules/overrides.tenantStatusPageContents*/ -}}
 <!DOCTYPE html>
 <html>
 <head>
diff --git a/modules/overrides/tenants.gohtml b/modules/overrides/tenants.gohtml
index 53424864f6a..cb981687a37 100644
--- a/modules/overrides/tenants.gohtml
+++ b/modules/overrides/tenants.gohtml
@@ -1,4 +1,4 @@
-{{- /*gotype: github.com/grafana/mimir/pkg/storegateway.tenantsPageContents*/ -}}
+{{- /*gotype: github.com/grafana/tempo/modules/overrides.tenantsPageContents*/ -}}
 <!DOCTYPE html>
 <html>
 <head>
diff --git a/modules/overrides/user_configurable_overrides.go b/modules/overrides/user_configurable_overrides.go
index e77075eed8d..6f3b52f29f0 100644
--- a/modules/overrides/user_configurable_overrides.go
+++ b/modules/overrides/user_configurable_overrides.go
@@ -16,6 +16,7 @@ import (
 	"github.com/opentracing/opentracing-go"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promauto"
+	"golang.org/x/exp/maps"
 	"golang.org/x/exp/slices"
 	"gopkg.in/yaml.v2"
 
@@ -199,11 +200,7 @@ func (o *userConfigurableOverridesManager) setTenantLimit(userID string, limits
 }
 
 func (o *userConfigurableOverridesManager) GetTenantIDs() []string {
-	var ids []string
-	for tenant := range o.getAllTenantLimits() {
-		ids = append(ids, tenant)
-	}
-	return ids
+	return maps.Keys(o.getAllTenantLimits())
 }
 
 func (o *userConfigurableOverridesManager) Forwarders(userID string) []string {

From 34d280291f20946baae2cef9540cbd3580d2232e Mon Sep 17 00:00:00 2001
From: Koenraad Verheyden <koenraad.verheyden@grafana.com>
Date: Thu, 1 Feb 2024 16:10:50 +0100
Subject: [PATCH 8/8] Specify source of runtime overrides

---
 .../overrides/overrides_tenant_status_http.go | 49 +++++++++++++------
 modules/overrides/tenant_status.gohtml        |  1 +
 2 files changed, 34 insertions(+), 16 deletions(-)

diff --git a/modules/overrides/overrides_tenant_status_http.go b/modules/overrides/overrides_tenant_status_http.go
index 279df0ecf9b..376b24ca61c 100644
--- a/modules/overrides/overrides_tenant_status_http.go
+++ b/modules/overrides/overrides_tenant_status_http.go
@@ -8,6 +8,7 @@ import (
 	"time"
 
 	"github.com/gorilla/mux"
+	"golang.org/x/exp/slices"
 	"gopkg.in/yaml.v2"
 
 	"github.com/grafana/tempo/pkg/util"
@@ -20,52 +21,68 @@ var tenantStatusTemplate = template.Must(template.New("webpage").Parse(tenantSta
 type tenantStatusPageContents struct {
 	Now                       time.Time `json:"now"`
 	Tenant                    string    `json:"tenant"`
-	RuntimeOverrides          string    `json:"runtime_overrides"`
 	UserConfigurableOverrides string    `json:"user_configurable_overrides"`
+	RuntimeOverrides          string    `json:"runtime_overrides"`
+	RuntimeOverridesSource    string    `json:"using_default_or_wildcard_runtime_overrides"`
 }
 
 func TenantStatusHandler(o Interface) http.HandlerFunc {
 	return func(w http.ResponseWriter, req *http.Request) {
+		page := tenantStatusPageContents{
+			Now: time.Now(),
+		}
+
 		vars := mux.Vars(req)
 
-		tenantID := vars["tenant"]
-		if tenantID == "" {
+		page.Tenant = vars["tenant"]
+		if page.Tenant == "" {
 			util.WriteTextResponse(w, "Tenant ID can't be empty")
 			return
 		}
 
 		// runtime overrides
-		overrides := o.GetRuntimeOverridesFor(tenantID)
+		overrides := o.GetRuntimeOverridesFor(page.Tenant)
 		runtimeOverrides, err := yaml.Marshal(overrides)
 		if err != nil {
 			util.WriteTextResponse(w, fmt.Sprintf("Marshalling runtime overrides failed: %s", err))
 			return
 		}
+		page.RuntimeOverrides = string(runtimeOverrides)
 
-		// user-configurable overrides
-		var userConfigurableOverrides string
+		var runtimeTenants []string
+		switch o := o.(type) {
+		case *runtimeConfigOverridesManager:
+			runtimeTenants = o.GetTenantIDs()
+		case *userConfigurableOverridesManager:
+			runtimeTenants = o.Interface.GetTenantIDs()
+		default:
+			util.WriteTextResponse(w, "Internal error happened when retrieving runtime overrides")
+		}
+		if slices.Contains(runtimeTenants, page.Tenant) {
+			page.RuntimeOverridesSource = page.Tenant
+		} else if slices.Contains(runtimeTenants, wildcardTenant) {
+			page.RuntimeOverridesSource = wildcardTenant
+		} else {
+			page.RuntimeOverridesSource = "default overrides"
+		}
 
+		// user-configurable overrides
 		if userConfigOverridesManager, ok := o.(*userConfigurableOverridesManager); ok {
-			overrides := userConfigOverridesManager.getTenantLimits(tenantID)
+			overrides := userConfigOverridesManager.getTenantLimits(page.Tenant)
 			if overrides != nil {
 				marshalledOverrides, err := yaml.Marshal(overrides)
 				if err != nil {
 					util.WriteTextResponse(w, fmt.Sprintf("Marshalling user-configurable overrides failed: %s", err))
 					return
 				}
-				userConfigurableOverrides = string(marshalledOverrides)
+				page.UserConfigurableOverrides = string(marshalledOverrides)
 			} else {
-				userConfigurableOverrides = "No user-configurable overrides set"
+				page.UserConfigurableOverrides = "No user-configurable overrides set"
 			}
 		} else {
-			userConfigurableOverrides = "User-configurable overrides are not enabled"
+			page.UserConfigurableOverrides = "User-configurable overrides are not enabled"
 		}
 
-		util.RenderHTTPResponse(w, tenantStatusPageContents{
-			Now:                       time.Now(),
-			Tenant:                    tenantID,
-			RuntimeOverrides:          string(runtimeOverrides),
-			UserConfigurableOverrides: userConfigurableOverrides,
-		}, tenantStatusTemplate, req)
+		util.RenderHTTPResponse(w, page, tenantStatusTemplate, req)
 	}
 }
diff --git a/modules/overrides/tenant_status.gohtml b/modules/overrides/tenant_status.gohtml
index 2f46da5cae6..0e0c4b1f9bd 100644
--- a/modules/overrides/tenant_status.gohtml
+++ b/modules/overrides/tenant_status.gohtml
@@ -12,6 +12,7 @@
 <h3>User-configurable overrides</h3>
 <pre><code>{{ .UserConfigurableOverrides }}</code></pre>
 <h3>Runtime overrides</h3>
+<p>Source of runtime overrides: <strong>{{ .RuntimeOverridesSource }}</strong></p>
 <pre><code>{{ .RuntimeOverrides }}</code></pre>
 </body>
 </html>