fix: use cluster metrics-server instead of plugin metrics endpoints

README.md

@@ -11,12 +11,12 @@ The Helm chart deploys one DaemonSet (`kks-provider-plugin-provider`) that runs
 
 ## Helm Chart
 
-Chart path: `charts/kks-provider-plugin`
+Chart URL: `https://github.com/kubelancloud/kks-provider-plugin/releases/download/v1.0.0/kks-provider-plugin-1.0.0.tgz`
 
 Example install:
 
 ```bash
-helm install kks-provider-plugin ./charts/kks-provider-plugin \
+helm install kks-provider-plugin https://github.com/kubelancloud/kks-provider-plugin/releases/download/v1.0.0/kks-provider-plugin-1.0.0.tgz \
   --namespace kube-system \
   --create-namespace \
   --set lb.serverURL=https://lb.example.kloud.team \
@@ -33,13 +33,11 @@ The container/binary entrypoint is `kks-provider` and exposes:
 - `kks-provider lb-controller`
 - `kks-provider lb-speaker`
 
-Each command also starts a Prometheus metrics endpoint on `/metrics` by default:
+For cluster-wide pod/node resource metrics (for `kubectl top` and autoscaling), deploy Kubernetes metrics-server once per cluster, for example:
 
-- `kks-provider csi` on `:10080`
+```bash
-- `kks-provider lb-controller` on `:10081`
+kubectl apply -f https://github.com/kubernetes-sigs/metrics-server/releases/latest/download/high-availability-1.21+.yaml
-- `kks-provider lb-speaker` on `:10082`
+```
-
-You can override the bind address with `--metrics-bind-address` (or disable metrics with `--metrics-bind-address=off`).
 
 ## Image
 

charts/kks-provider-plugin/templates/provider-daemonset.yaml

@@ -57,8 +57,6 @@ spec:
             - csi
             - --config-file
             - /etc/kloud-csi/driver.hcl
-            - --metrics-bind-address
-            - {{ .Values.metrics.csi.bindAddress | quote }}
           env:
             - name: KKS_CSI_SERVER_URL
               value: {{ .Values.csi.serverURL | quote }}
@@ -157,8 +155,6 @@ spec:
           command:
             - /kks-provider
             - lb-controller
-            - --metrics-bind-address
-            - {{ .Values.metrics.lbController.bindAddress | quote }}
           env:
             - name: KLOUD_LB_API_URL
               value: {{ .Values.lb.serverURL | quote }}
@@ -181,8 +177,6 @@ spec:
           command:
             - /kks-provider
             - lb-speaker
-            - --metrics-bind-address
-            - {{ .Values.metrics.lbSpeaker.bindAddress | quote }}
           securityContext:
             capabilities:
               add:

charts/kks-provider-plugin/values.yaml

@@ -90,11 +90,3 @@ podAnnotations: {}
 
 nodeSelector: {}
 affinity: {}
-
-metrics:
-  csi:
-    bindAddress: ":10080"
-  lbController:
-    bindAddress: ":10081"
-  lbSpeaker:
-    bindAddress: ":10082"

go.mod

@@ -5,7 +5,6 @@ go 1.24.5
 require (
 	github.com/container-storage-interface/spec v1.9.0
 	github.com/hashicorp/hcl/v2 v2.24.0
-	github.com/prometheus/client_golang v1.22.0
 	github.com/spf13/cobra v1.9.1
 	github.com/vishvananda/netlink v1.3.1
 	go.uber.org/zap v1.27.0
@@ -21,8 +20,6 @@ require (
 require (
 	github.com/agext/levenshtein v1.2.1 // indirect
 	github.com/apparentlymart/go-textseg/v15 v15.0.0 // indirect
-	github.com/beorn7/perks v1.0.1 // indirect
-	github.com/cespare/xxhash/v2 v2.3.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/emicklei/go-restful/v3 v3.11.0 // indirect
 	github.com/fxamacker/cbor/v2 v2.7.0 // indirect
@@ -45,9 +42,6 @@ require (
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
-	github.com/prometheus/client_model v0.6.1 // indirect
-	github.com/prometheus/common v0.62.0 // indirect
-	github.com/prometheus/procfs v0.15.1 // indirect
 	github.com/spf13/pflag v1.0.6 // indirect
 	github.com/vishvananda/netns v0.0.5 // indirect
 	github.com/x448/float16 v0.8.4 // indirect

go.sum

@@ -2,10 +2,6 @@ github.com/agext/levenshtein v1.2.1 h1:QmvMAjj2aEICytGiWzmxoE0x2KZvE0fvmqMOfy2tj
 github.com/agext/levenshtein v1.2.1/go.mod h1:JEDfjyjHDjOF/1e4FlBE/PkbqA9OfWu2ki2W0IB5558=
 github.com/apparentlymart/go-textseg/v15 v15.0.0 h1:uYvfpb3DyLSCGWnctWKGj857c6ew1u1fNQOlOtuGxQY=
 github.com/apparentlymart/go-textseg/v15 v15.0.0/go.mod h1:K8XmNZdhEBkdlyDdvbmmsvpAG721bKi0joRfFdHIWJ4=
-github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
-github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
-github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
-github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/container-storage-interface/spec v1.9.0 h1:zKtX4STsq31Knz3gciCYCi1SXtO2HJDecIjDVboYavY=
 github.com/container-storage-interface/spec v1.9.0/go.mod h1:ZfDu+3ZRyeVqxZM0Ds19MVLkN2d1XJ5MAfi1L3VjlT0=
 github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
@@ -55,8 +51,6 @@ github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnr
 github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
 github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
-github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo=
-github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ=
 github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
 github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
 github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
@@ -64,8 +58,6 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
-github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
-github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
 github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
 github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
 github.com/mitchellh/go-wordwrap v1.0.1 h1:TLuKupo69TCn6TQSyGxwI1EblZZEsQ0vMlAFQflz0v0=
@@ -87,14 +79,6 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/prometheus/client_golang v1.22.0 h1:rb93p9lokFEsctTys46VnV1kLCDpVZ0a/Y92Vm0Zc6Q=
-github.com/prometheus/client_golang v1.22.0/go.mod h1:R7ljNsLXhuQXYZYtw6GAE9AZg8Y7vEW5scdCXrWRXC0=
-github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E=
-github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY=
-github.com/prometheus/common v0.62.0 h1:xasJaQlnWAeyHdUBeGjXmutelfJHWMRr+Fg4QszZ2Io=
-github.com/prometheus/common v0.62.0/go.mod h1:vyBcEuLSvWos9B1+CyL7JZ2up+uFzXhkqml0W5zIY1I=
-github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc=
-github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk=
 github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII=
 github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=

main.go

@@ -2,13 +2,9 @@ package main
 
 import (
 	"context"
-	"errors"
 	"fmt"
-	"net"
-	"net/http"
 	"os"
 	"os/signal"
-	"strings"
 	"syscall"
 	"time"
 
@@ -17,7 +13,6 @@ import (
 	"github.com/KubelanCloud/kks-provider-plugin/pkg/kloudlb/controller"
 	"github.com/KubelanCloud/kks-provider-plugin/pkg/kloudlb/speaker"
 	lbapi "github.com/KubelanCloud/kks-provider-plugin/pkg/lb/api"
-	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"github.com/spf13/cobra"
 	"go.uber.org/zap"
 	"k8s.io/client-go/kubernetes"
@@ -39,7 +34,7 @@ func main() {
 		Long:  "Runs the in-cluster provider components for CSI and LoadBalancer integration with kks management services.",
 	}
 
-	rootCmd.AddCommand(csiCmd(logger), lbControllerCmd(logger), lbSpeakerCmd(logger))
+	rootCmd.AddCommand(csiCmd(logger), lbControllerCmd(), lbSpeakerCmd())
 
 	if err := rootCmd.Execute(); err != nil {
 		os.Exit(1)
@@ -48,26 +43,22 @@ func main() {
 
 func csiCmd(logger *zap.Logger) *cobra.Command {
 	configPath := "csi.hcl"
-	metricsBindAddress := ":10080"
 	cmd := &cobra.Command{
 		Use:   "csi",
 		Short: "Run the Kloud CSI driver",
 		RunE: func(cmd *cobra.Command, args []string) error {
-			return runCSI(cmd, configPath, metricsBindAddress, logger)
+			return runCSI(cmd, configPath, logger)
 		},
 	}
 	cmd.Flags().StringVarP(&configPath, "config-file", "c", "csi.hcl", "Path to driver config (optional when using env vars)")
-	cmd.Flags().StringVar(&metricsBindAddress, "metrics-bind-address", ":10080", "Address to bind metrics server (set to 'off' to disable)")
 	return cmd
 }
 
-func lbControllerCmd(logger *zap.Logger) *cobra.Command {
+func lbControllerCmd() *cobra.Command {
 	var (
-		apiURL             string
+		apiURL string
-		token              string
+		token  string
-		metricsBindAddress string
 	)
-	metricsBindAddress = ":10081"
 	cmd := &cobra.Command{
 		Use:   "lb-controller",
 		Short: "Run the KloudLB Service controller",
@@ -98,20 +89,15 @@ func lbControllerCmd(logger *zap.Logger) *cobra.Command {
 
 			ctx, cancel := signalContext(cmd.Context())
 			defer cancel()
-			if err := startMetricsServer(ctx, logger, metricsBindAddress); err != nil {
-				return err
-			}
 			return ctrl.Run(ctx, 2)
 		},
 	}
 	cmd.Flags().StringVar(&apiURL, "api-url", "", "KKS LoadBalancer API base URL")
 	cmd.Flags().StringVar(&token, "access-token", "", "Cluster LB access token")
-	cmd.Flags().StringVar(&metricsBindAddress, "metrics-bind-address", ":10081", "Address to bind metrics server (set to 'off' to disable)")
 	return cmd
 }
 
-func lbSpeakerCmd(logger *zap.Logger) *cobra.Command {
+func lbSpeakerCmd() *cobra.Command {
-	metricsBindAddress := ":10082"
 	cmd := &cobra.Command{
 		Use:   "lb-speaker",
 		Short: "Run the KloudLB L2 speaker",
@@ -132,17 +118,13 @@ func lbSpeakerCmd(logger *zap.Logger) *cobra.Command {
 
 			ctx, cancel := signalContext(cmd.Context())
 			defer cancel()
-			if err := startMetricsServer(ctx, logger, metricsBindAddress); err != nil {
-				return err
-			}
 			return spk.Run(ctx)
 		},
 	}
-	cmd.Flags().StringVar(&metricsBindAddress, "metrics-bind-address", ":10082", "Address to bind metrics server (set to 'off' to disable)")
 	return cmd
 }
 
-func runCSI(cmd *cobra.Command, configPath, metricsBindAddress string, logger *zap.Logger) error {
+func runCSI(cmd *cobra.Command, configPath string, logger *zap.Logger) error {
 	cfg, err := config.LoadClient(configPath)
 	if err != nil {
 		return fmt.Errorf("load config: %w", err)
@@ -150,57 +132,11 @@ func runCSI(cmd *cobra.Command, configPath, metricsBindAddress string, logger *z
 
 	ctx, cancel := signalContext(cmd.Context())
 	defer cancel()
-	if err := startMetricsServer(ctx, logger, metricsBindAddress); err != nil {
-		return err
-	}
 
 	logger.Sugar().Infof("loaded csi driver config from %s", configPath)
 	return driver.Run(ctx, cfg, logger)
 }
 
-func startMetricsServer(ctx context.Context, logger *zap.Logger, bindAddress string) error {
-	bindAddress = strings.TrimSpace(bindAddress)
-	if bindAddress == "" || strings.EqualFold(bindAddress, "off") {
-		logger.Info("metrics server disabled")
-		return nil
-	}
-
-	listener, err := net.Listen("tcp", bindAddress)
-	if err != nil {
-		return fmt.Errorf("listen metrics server on %s: %w", bindAddress, err)
-	}
-
-	server := &http.Server{
-		Addr:              bindAddress,
-		Handler:           metricsMux(),
-		ReadHeaderTimeout: 5 * time.Second,
-	}
-
-	go func() {
-		<-ctx.Done()
-		shutdownCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
-		defer cancel()
-		if err := server.Shutdown(shutdownCtx); err != nil && !errors.Is(err, http.ErrServerClosed) {
-			logger.Error("failed to shutdown metrics server", zap.Error(err))
-		}
-	}()
-
-	go func() {
-		logger.Sugar().Infof("metrics server listening on %s", bindAddress)
-		if err := server.Serve(listener); err != nil && !errors.Is(err, http.ErrServerClosed) {
-			logger.Error("metrics server stopped unexpectedly", zap.Error(err))
-		}
-	}()
-
-	return nil
-}
-
-func metricsMux() *http.ServeMux {
-	mux := http.NewServeMux()
-	mux.Handle("/metrics", promhttp.Handler())
-	return mux
-}
-
 func restConfig() (*rest.Config, error) {
 	if cfg, err := rest.InClusterConfig(); err == nil {
 		return cfg, nil