Enable metrics provider

Author: jonathan-innis

pkg/config/controller.go

@@ -20,6 +20,7 @@ import (
 	"time"
 
 	"github.com/go-logr/logr"
+	"sigs.k8s.io/controller-runtime/pkg/metrics"
 )
 
 // Controller contains configuration options for controllers. It only includes options
@@ -66,6 +67,10 @@ type Controller struct {
 	// Note: This flag is disabled by default until a future version. It's currently in beta.
 	UsePriorityQueue *bool
 
+	// MetricsProvider allows users to override the location where controller metrics are emitted.
+	// By default, metrics are emitted to a pre-configured Prometheus registry
+	MetricsProvider metrics.ControllerMetricsProvider
+
 	// Logger is the logger controllers should use.
 	Logger logr.Logger
 }

pkg/controller/controller.go

@@ -25,6 +25,7 @@ import (
 	"k8s.io/client-go/util/workqueue"
 	"k8s.io/klog/v2"
 	"k8s.io/utils/ptr"
+	"sigs.k8s.io/controller-runtime/pkg/metrics"
 
 	"sigs.k8s.io/controller-runtime/pkg/config"
 	"sigs.k8s.io/controller-runtime/pkg/controller/priorityqueue"
@@ -84,6 +85,10 @@ type TypedOptions[request comparable] struct {
 	// Logger will be used to build a default LogConstructor if unset.
 	Logger logr.Logger
 
+	// MetricsProvider allows users to override the location where controller metrics are emitted.
+	// By default, metrics are emitted to a pre-configured Prometheus registry
+	MetricsProvider metrics.ControllerMetricsProvider
+
 	// LogConstructor is used to construct a logger used for this controller and passed
 	// to each reconciliation via the context field.
 	LogConstructor func(request *request) logr.Logger
@@ -101,6 +106,10 @@ func (options *TypedOptions[request]) DefaultFromConfig(config config.Controller
 		options.Logger = config.Logger
 	}
 
+	if options.MetricsProvider == nil {
+		options.MetricsProvider = config.MetricsProvider
+	}
+
 	if options.SkipNameValidation == nil {
 		options.SkipNameValidation = config.SkipNameValidation
 	}
@@ -196,6 +205,10 @@ func NewTypedUnmanaged[request comparable](name string, options TypedOptions[req
 		}
 	}
 
+	if options.MetricsProvider == nil {
+		options.MetricsProvider = metrics.NewPrometheusProvider()
+	}
+
 	if options.LogConstructor == nil {
 		log := options.Logger.WithValues(
 			"controller", name,
@@ -250,6 +263,7 @@ func NewTypedUnmanaged[request comparable](name string, options TypedOptions[req
 		MaxConcurrentReconciles: options.MaxConcurrentReconciles,
 		CacheSyncTimeout:        options.CacheSyncTimeout,
 		Name:                    name,
+		MetricsProvider:         options.MetricsProvider,
 		LogConstructor:          options.LogConstructor,
 		RecoverPanic:            options.RecoverPanic,
 		LeaderElected:           options.NeedLeaderElection,

pkg/internal/controller/controller.go

@@ -30,9 +30,9 @@ import (
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	"k8s.io/apimachinery/pkg/util/uuid"
 	"k8s.io/client-go/util/workqueue"
+	"sigs.k8s.io/controller-runtime/pkg/metrics"
 
 	"sigs.k8s.io/controller-runtime/pkg/controller/priorityqueue"
-	ctrlmetrics "sigs.k8s.io/controller-runtime/pkg/internal/controller/metrics"
 	logf "sigs.k8s.io/controller-runtime/pkg/log"
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
 	"sigs.k8s.io/controller-runtime/pkg/source"
@@ -89,6 +89,9 @@ type Controller[request comparable] struct {
 	// outside the context of a reconciliation.
 	LogConstructor func(request *request) logr.Logger
 
+	// MetricsProvider is used to route metrics that are fired due to controller reconciles
+	MetricsProvider metrics.ControllerMetricsProvider
+
 	// RecoverPanic indicates whether the panic caused by reconcile should be recovered.
 	// Defaults to true.
 	RecoverPanic *bool
@@ -101,7 +104,7 @@ type Controller[request comparable] struct {
 func (c *Controller[request]) Reconcile(ctx context.Context, req request) (_ reconcile.Result, err error) {
 	defer func() {
 		if r := recover(); r != nil {
-			ctrlmetrics.ReconcilePanics.WithLabelValues(c.Name).Inc()
+			c.MetricsProvider.ReconcilePanics().Inc(map[string]string{labelKeyController: c.Name})
 
 			if c.RecoverPanic == nil || *c.RecoverPanic {
 				for _, fn := range utilruntime.PanicHandlers {
@@ -294,30 +297,32 @@ func (c *Controller[request]) processNextWorkItem(ctx context.Context) bool {
 	// period.
 	defer c.Queue.Done(obj)
 
-	ctrlmetrics.ActiveWorkers.WithLabelValues(c.Name).Add(1)
-	defer ctrlmetrics.ActiveWorkers.WithLabelValues(c.Name).Add(-1)
+	c.MetricsProvider.ActiveWorkers().Add(map[string]string{labelKeyController: c.Name}, 1)
+	defer c.MetricsProvider.ActiveWorkers().Add(map[string]string{labelKeyController: c.Name}, -1)
 
 	c.reconcileHandler(ctx, obj, priority)
 	return true
 }
 
 const (
-	labelError        = "error"
-	labelRequeueAfter = "requeue_after"
-	labelRequeue      = "requeue"
-	labelSuccess      = "success"
+	labelKeyController = "controller"
+	labelKeyResult     = "result"
+	labelError         = "error"
+	labelRequeueAfter  = "requeue_after"
+	labelRequeue       = "requeue"
+	labelSuccess       = "success"
 )
 
 func (c *Controller[request]) initMetrics() {
-	ctrlmetrics.ReconcileTotal.WithLabelValues(c.Name, labelError).Add(0)
-	ctrlmetrics.ReconcileTotal.WithLabelValues(c.Name, labelRequeueAfter).Add(0)
-	ctrlmetrics.ReconcileTotal.WithLabelValues(c.Name, labelRequeue).Add(0)
-	ctrlmetrics.ReconcileTotal.WithLabelValues(c.Name, labelSuccess).Add(0)
-	ctrlmetrics.ReconcileErrors.WithLabelValues(c.Name).Add(0)
-	ctrlmetrics.TerminalReconcileErrors.WithLabelValues(c.Name).Add(0)
-	ctrlmetrics.ReconcilePanics.WithLabelValues(c.Name).Add(0)
-	ctrlmetrics.WorkerCount.WithLabelValues(c.Name).Set(float64(c.MaxConcurrentReconciles))
-	ctrlmetrics.ActiveWorkers.WithLabelValues(c.Name).Set(0)
+	c.MetricsProvider.ReconcileTotal().Add(map[string]string{labelKeyController: c.Name, labelKeyResult: labelError}, 0)
+	c.MetricsProvider.ReconcileTotal().Add(map[string]string{labelKeyController: c.Name, labelKeyResult: labelRequeueAfter}, 0)
+	c.MetricsProvider.ReconcileTotal().Add(map[string]string{labelKeyController: c.Name, labelKeyResult: labelRequeue}, 0)
+	c.MetricsProvider.ReconcileTotal().Add(map[string]string{labelKeyController: c.Name, labelKeyResult: labelSuccess}, 0)
+	c.MetricsProvider.ReconcileErrors().Add(map[string]string{labelKeyController: c.Name}, 0)
+	c.MetricsProvider.TerminalReconcileErrors().Add(map[string]string{labelKeyController: c.Name}, 0)
+	c.MetricsProvider.ReconcilePanics().Add(map[string]string{labelKeyController: c.Name}, 0)
+	c.MetricsProvider.WorkerCount().Set(map[string]string{labelKeyController: c.Name}, float64(c.MaxConcurrentReconciles))
+	c.MetricsProvider.ActiveWorkers().Set(map[string]string{labelKeyController: c.Name}, 0)
 }
 
 func (c *Controller[request]) reconcileHandler(ctx context.Context, req request, priority int) {
@@ -341,12 +346,12 @@ func (c *Controller[request]) reconcileHandler(ctx context.Context, req request,
 	switch {
 	case err != nil:
 		if errors.Is(err, reconcile.TerminalError(nil)) {
-			ctrlmetrics.TerminalReconcileErrors.WithLabelValues(c.Name).Inc()
+			c.MetricsProvider.TerminalReconcileErrors().Inc(map[string]string{"controller": c.Name})
 		} else {
 			c.Queue.AddWithOpts(priorityqueue.AddOpts{RateLimited: true, Priority: priority}, req)
 		}
-		ctrlmetrics.ReconcileErrors.WithLabelValues(c.Name).Inc()
-		ctrlmetrics.ReconcileTotal.WithLabelValues(c.Name, labelError).Inc()
+		c.MetricsProvider.ReconcileErrors().Inc(map[string]string{labelKeyController: c.Name})
+		c.MetricsProvider.ReconcileTotal().Inc(map[string]string{labelKeyController: c.Name, labelKeyResult: labelError})
 		if !result.IsZero() {
 			log.Info("Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler")
 		}
@@ -359,17 +364,17 @@ func (c *Controller[request]) reconcileHandler(ctx context.Context, req request,
 		// to result.RequestAfter
 		c.Queue.Forget(req)
 		c.Queue.AddWithOpts(priorityqueue.AddOpts{After: result.RequeueAfter, Priority: priority}, req)
-		ctrlmetrics.ReconcileTotal.WithLabelValues(c.Name, labelRequeueAfter).Inc()
+		c.MetricsProvider.ReconcileTotal().Inc(map[string]string{labelKeyController: c.Name, labelKeyResult: labelRequeueAfter})
 	case result.Requeue: //nolint: staticcheck // We have to handle it until it is removed
 		log.V(5).Info("Reconcile done, requeueing")
 		c.Queue.AddWithOpts(priorityqueue.AddOpts{RateLimited: true, Priority: priority}, req)
-		ctrlmetrics.ReconcileTotal.WithLabelValues(c.Name, labelRequeue).Inc()
+		c.MetricsProvider.ReconcileTotal().Inc(map[string]string{labelKeyController: c.Name, labelKeyResult: labelRequeue})
 	default:
 		log.V(5).Info("Reconcile successful")
 		// Finally, if no error occurs we Forget this item so it does not
 		// get queued again until another change happens.
 		c.Queue.Forget(req)
-		ctrlmetrics.ReconcileTotal.WithLabelValues(c.Name, labelSuccess).Inc()
+		c.MetricsProvider.ReconcileTotal().Inc(map[string]string{labelKeyController: c.Name, labelKeyResult: labelSuccess})
 	}
 }
 
@@ -380,7 +385,7 @@ func (c *Controller[request]) GetLogger() logr.Logger {
 
 // updateMetrics updates prometheus metrics within the controller.
 func (c *Controller[request]) updateMetrics(reconcileTime time.Duration) {
-	ctrlmetrics.ReconcileTime.WithLabelValues(c.Name).Observe(reconcileTime.Seconds())
+	c.MetricsProvider.ReconcileTime().Observe(map[string]string{labelKeyController: c.Name}, reconcileTime.Seconds())
 }
 
 // ReconcileIDFromContext gets the reconcileID from the current context.

pkg/internal/controller/controller_test.go

@@ -43,6 +43,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/handler"
 	ctrlmetrics "sigs.k8s.io/controller-runtime/pkg/internal/controller/metrics"
 	"sigs.k8s.io/controller-runtime/pkg/internal/log"
+	"sigs.k8s.io/controller-runtime/pkg/metrics"
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
 	"sigs.k8s.io/controller-runtime/pkg/source"
 )
@@ -75,6 +76,7 @@ var _ = Describe("controller", func() {
 			NewQueue: func(string, workqueue.TypedRateLimiter[reconcile.Request]) workqueue.TypedRateLimitingInterface[reconcile.Request] {
 				return queue
 			},
+			MetricsProvider: metrics.NewPrometheusProvider(),
 			LogConstructor: func(_ *reconcile.Request) logr.Logger {
 				return log.RuntimeLog.WithName("controller").WithName("test")
 			},
@@ -354,6 +356,7 @@ var _ = Describe("controller", func() {
 				NewQueue: func(string, workqueue.TypedRateLimiter[TestRequest]) workqueue.TypedRateLimitingInterface[TestRequest] {
 					return queue
 				},
+				MetricsProvider: metrics.NewPrometheusProvider(),
 				LogConstructor: func(*TestRequest) logr.Logger {
 					return log.RuntimeLog.WithName("controller").WithName("test")
 				},

pkg/internal/controller/metrics/metrics.go

@@ -20,8 +20,6 @@ import (
 	"time"
 
 	"github.com/prometheus/client_golang/prometheus"
-	"github.com/prometheus/client_golang/prometheus/collectors"
-	"sigs.k8s.io/controller-runtime/pkg/metrics"
 )
 
 var (
@@ -81,19 +79,3 @@ var (
 		Help: "Number of currently used workers per controller",
 	}, []string{"controller"})
 )
-
-func init() {
-	metrics.Registry.MustRegister(
-		ReconcileTotal,
-		ReconcileErrors,
-		TerminalReconcileErrors,
-		ReconcilePanics,
-		ReconcileTime,
-		WorkerCount,
-		ActiveWorkers,
-		// expose process metrics like CPU, Memory, file descriptor usage etc.
-		collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}),
-		// expose all Go runtime metrics like GC stats, memory stats etc.
-		collectors.NewGoCollector(collectors.WithGoCollectorRuntimeMetrics(collectors.MetricsAll)),
-	)
-}

pkg/manager/internal.go

@@ -35,7 +35,6 @@ import (
 	"k8s.io/client-go/tools/leaderelection"
 	"k8s.io/client-go/tools/leaderelection/resourcelock"
 	"k8s.io/client-go/tools/record"
-
 	"sigs.k8s.io/controller-runtime/pkg/cache"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/cluster"

pkg/manager/manager.go

@@ -33,6 +33,7 @@ import (
 	"k8s.io/client-go/tools/leaderelection/resourcelock"
 	"k8s.io/client-go/tools/record"
 	"k8s.io/utils/ptr"
+	"sigs.k8s.io/controller-runtime/pkg/metrics"
 	metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
 
 	"sigs.k8s.io/controller-runtime/pkg/cache"
@@ -223,6 +224,10 @@ type Options struct {
 	// Metrics are the metricsserver.Options that will be used to create the metricsserver.Server.
 	Metrics metricsserver.Options
 
+	// LeaderEelectionMetricProvider allows users to override the location where leader election metrics are emitted.
+	// By default, metrics are emitted to a pre-configured Prometheus registry
+	LeaderElectionMetricProvider metrics.LeaderElectionMetricsProvider
+
 	// HealthProbeBindAddress is the TCP address that the controller should bind to
 	// for serving health probes
 	// It can be set to "0" or "" to disable serving the health probe.
@@ -401,6 +406,11 @@ func New(config *rest.Config, options Options) (Manager, error) {
 	if err != nil {
 		return nil, err
 	}
+	leaderElectionMetricsProvider := options.LeaderElectionMetricProvider
+	if leaderElectionMetricsProvider == nil {
+		leaderElectionMetricsProvider = metrics.NewPrometheusProvider()
+	}
+	metrics.SetLeaderElectionProvider(leaderElectionMetricsProvider)
 
 	// Create health probes listener. This will throw an error if the bind
 	// address is invalid or already in use.

pkg/metrics/leaderelection.go

@@ -1,47 +1,34 @@
 package metrics
 
 import (
-	"github.com/prometheus/client_golang/prometheus"
 	"k8s.io/client-go/tools/leaderelection"
 )
 
-// This file is copied and adapted from k8s.io/component-base/metrics/prometheus/clientgo/leaderelection
-// which registers metrics to the k8s legacy Registry. We require very
-// similar functionality, but must register metrics to a different Registry.
-
-var (
-	leaderGauge = prometheus.NewGaugeVec(prometheus.GaugeOpts{
-		Name: "leader_election_master_status",
-		Help: "Gauge of if the reporting system is master of the relevant lease, 0 indicates backup, 1 indicates master. 'name' is the string used to identify the lease. Please make sure to group by name.",
-	}, []string{"name"})
-
-	leaderSlowpathCounter = prometheus.NewCounterVec(prometheus.CounterOpts{
-		Name: "leader_election_slowpath_total",
-		Help: "Total number of slow path exercised in renewing leader leases. 'name' is the string used to identify the lease. Please make sure to group by name.",
-	}, []string{"name"})
-)
-
-func init() {
-	Registry.MustRegister(leaderGauge)
-	leaderelection.SetProvider(leaderelectionMetricsProvider{})
+// SetLeaderElectionProvider sets the leader election provider leveraged by client-go
+func SetLeaderElectionProvider(provider LeaderElectionMetricsProvider) {
+	leaderelection.SetProvider(leaderElectionMetricsProvider{provider: provider})
 }
 
-type leaderelectionMetricsProvider struct{}
+type leaderElectionMetricsProvider struct {
+	provider LeaderElectionMetricsProvider
+}
 
-func (leaderelectionMetricsProvider) NewLeaderMetric() leaderelection.LeaderMetric {
-	return leaderElectionPrometheusAdapter{}
+func (l leaderElectionMetricsProvider) NewLeaderMetric() leaderelection.LeaderMetric {
+	return leaderElectionMetricAdapter(l)
 }
 
-type leaderElectionPrometheusAdapter struct{}
+type leaderElectionMetricAdapter struct {
+	provider LeaderElectionMetricsProvider
+}
 
-func (s leaderElectionPrometheusAdapter) On(name string) {
-	leaderGauge.WithLabelValues(name).Set(1.0)
+func (l leaderElectionMetricAdapter) On(name string) {
+	l.provider.LeaderGauge().Set(map[string]string{"name": name}, 1)
 }
 
-func (s leaderElectionPrometheusAdapter) Off(name string) {
-	leaderGauge.WithLabelValues(name).Set(0.0)
+func (l leaderElectionMetricAdapter) Off(name string) {
+	l.provider.LeaderGauge().Set(map[string]string{"name": name}, 0)
 }
 
-func (leaderElectionPrometheusAdapter) SlowpathExercised(name string) {
-	leaderSlowpathCounter.WithLabelValues(name).Inc()
+func (l leaderElectionMetricAdapter) SlowpathExercised(name string) {
+	l.provider.SlowpathExercised().Inc(map[string]string{"name": name})
 }