diff --git a/README.md b/README.md index db04380..dfb8fdc 100644 --- a/README.md +++ b/README.md @@ -11,12 +11,12 @@ The Helm chart deploys one DaemonSet (`kks-provider-plugin-provider`) that runs ## Helm Chart -Chart path: `charts/kks-provider-plugin` +Chart URL: `https://github.com/kubelancloud/kks-provider-plugin/releases/download/v1.0.0/kks-provider-plugin-1.0.0.tgz` Example install: ```bash -helm install kks-provider-plugin ./charts/kks-provider-plugin \ +helm install kks-provider-plugin https://github.com/kubelancloud/kks-provider-plugin/releases/download/v1.0.0/kks-provider-plugin-1.0.0.tgz \ --namespace kube-system \ --create-namespace \ --set lb.serverURL=https://lb.example.kloud.team \ @@ -33,13 +33,11 @@ The container/binary entrypoint is `kks-provider` and exposes: - `kks-provider lb-controller` - `kks-provider lb-speaker` -Each command also starts a Prometheus metrics endpoint on `/metrics` by default: +For cluster-wide pod/node resource metrics (for `kubectl top` and autoscaling), deploy Kubernetes metrics-server once per cluster, for example: -- `kks-provider csi` on `:10080` -- `kks-provider lb-controller` on `:10081` -- `kks-provider lb-speaker` on `:10082` - -You can override the bind address with `--metrics-bind-address` (or disable metrics with `--metrics-bind-address=off`). +```bash +kubectl apply -f https://github.com/kubernetes-sigs/metrics-server/releases/latest/download/high-availability-1.21+.yaml +``` ## Image diff --git a/charts/kks-provider-plugin/templates/provider-daemonset.yaml b/charts/kks-provider-plugin/templates/provider-daemonset.yaml index 7aabb45..3e58fdd 100644 --- a/charts/kks-provider-plugin/templates/provider-daemonset.yaml +++ b/charts/kks-provider-plugin/templates/provider-daemonset.yaml @@ -57,8 +57,6 @@ spec: - csi - --config-file - /etc/kloud-csi/driver.hcl - - --metrics-bind-address - - {{ .Values.metrics.csi.bindAddress | quote }} env: - name: KKS_CSI_SERVER_URL value: {{ .Values.csi.serverURL | quote }} @@ -157,8 +155,6 @@ spec: command: - /kks-provider - lb-controller - - --metrics-bind-address - - {{ .Values.metrics.lbController.bindAddress | quote }} env: - name: KLOUD_LB_API_URL value: {{ .Values.lb.serverURL | quote }} @@ -181,8 +177,6 @@ spec: command: - /kks-provider - lb-speaker - - --metrics-bind-address - - {{ .Values.metrics.lbSpeaker.bindAddress | quote }} securityContext: capabilities: add: diff --git a/charts/kks-provider-plugin/values.yaml b/charts/kks-provider-plugin/values.yaml index e62ed2f..6f768f7 100644 --- a/charts/kks-provider-plugin/values.yaml +++ b/charts/kks-provider-plugin/values.yaml @@ -90,11 +90,3 @@ podAnnotations: {} nodeSelector: {} affinity: {} - -metrics: - csi: - bindAddress: ":10080" - lbController: - bindAddress: ":10081" - lbSpeaker: - bindAddress: ":10082" diff --git a/go.mod b/go.mod index ed26796..b9240f8 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,6 @@ go 1.24.5 require ( github.com/container-storage-interface/spec v1.9.0 github.com/hashicorp/hcl/v2 v2.24.0 - github.com/prometheus/client_golang v1.22.0 github.com/spf13/cobra v1.9.1 github.com/vishvananda/netlink v1.3.1 go.uber.org/zap v1.27.0 @@ -21,8 +20,6 @@ require ( require ( github.com/agext/levenshtein v1.2.1 // indirect github.com/apparentlymart/go-textseg/v15 v15.0.0 // indirect - github.com/beorn7/perks v1.0.1 // indirect - github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/emicklei/go-restful/v3 v3.11.0 // indirect github.com/fxamacker/cbor/v2 v2.7.0 // indirect @@ -45,9 +42,6 @@ require ( github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/pkg/errors v0.9.1 // indirect - github.com/prometheus/client_model v0.6.1 // indirect - github.com/prometheus/common v0.62.0 // indirect - github.com/prometheus/procfs v0.15.1 // indirect github.com/spf13/pflag v1.0.6 // indirect github.com/vishvananda/netns v0.0.5 // indirect github.com/x448/float16 v0.8.4 // indirect diff --git a/go.sum b/go.sum index 8867626..8cae36c 100644 --- a/go.sum +++ b/go.sum @@ -2,10 +2,6 @@ github.com/agext/levenshtein v1.2.1 h1:QmvMAjj2aEICytGiWzmxoE0x2KZvE0fvmqMOfy2tj github.com/agext/levenshtein v1.2.1/go.mod h1:JEDfjyjHDjOF/1e4FlBE/PkbqA9OfWu2ki2W0IB5558= github.com/apparentlymart/go-textseg/v15 v15.0.0 h1:uYvfpb3DyLSCGWnctWKGj857c6ew1u1fNQOlOtuGxQY= github.com/apparentlymart/go-textseg/v15 v15.0.0/go.mod h1:K8XmNZdhEBkdlyDdvbmmsvpAG721bKi0joRfFdHIWJ4= -github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= -github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= -github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/container-storage-interface/spec v1.9.0 h1:zKtX4STsq31Knz3gciCYCi1SXtO2HJDecIjDVboYavY= github.com/container-storage-interface/spec v1.9.0/go.mod h1:ZfDu+3ZRyeVqxZM0Ds19MVLkN2d1XJ5MAfi1L3VjlT0= github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= @@ -55,8 +51,6 @@ github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnr github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= -github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= @@ -64,8 +58,6 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= -github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/mitchellh/go-wordwrap v1.0.1 h1:TLuKupo69TCn6TQSyGxwI1EblZZEsQ0vMlAFQflz0v0= @@ -87,14 +79,6 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/prometheus/client_golang v1.22.0 h1:rb93p9lokFEsctTys46VnV1kLCDpVZ0a/Y92Vm0Zc6Q= -github.com/prometheus/client_golang v1.22.0/go.mod h1:R7ljNsLXhuQXYZYtw6GAE9AZg8Y7vEW5scdCXrWRXC0= -github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= -github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= -github.com/prometheus/common v0.62.0 h1:xasJaQlnWAeyHdUBeGjXmutelfJHWMRr+Fg4QszZ2Io= -github.com/prometheus/common v0.62.0/go.mod h1:vyBcEuLSvWos9B1+CyL7JZ2up+uFzXhkqml0W5zIY1I= -github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= -github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= diff --git a/main.go b/main.go index 5e65f7a..2fde4ca 100644 --- a/main.go +++ b/main.go @@ -2,13 +2,9 @@ package main import ( "context" - "errors" "fmt" - "net" - "net/http" "os" "os/signal" - "strings" "syscall" "time" @@ -17,7 +13,6 @@ import ( "github.com/KubelanCloud/kks-provider-plugin/pkg/kloudlb/controller" "github.com/KubelanCloud/kks-provider-plugin/pkg/kloudlb/speaker" lbapi "github.com/KubelanCloud/kks-provider-plugin/pkg/lb/api" - "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/spf13/cobra" "go.uber.org/zap" "k8s.io/client-go/kubernetes" @@ -39,7 +34,7 @@ func main() { Long: "Runs the in-cluster provider components for CSI and LoadBalancer integration with kks management services.", } - rootCmd.AddCommand(csiCmd(logger), lbControllerCmd(logger), lbSpeakerCmd(logger)) + rootCmd.AddCommand(csiCmd(logger), lbControllerCmd(), lbSpeakerCmd()) if err := rootCmd.Execute(); err != nil { os.Exit(1) @@ -48,26 +43,22 @@ func main() { func csiCmd(logger *zap.Logger) *cobra.Command { configPath := "csi.hcl" - metricsBindAddress := ":10080" cmd := &cobra.Command{ Use: "csi", Short: "Run the Kloud CSI driver", RunE: func(cmd *cobra.Command, args []string) error { - return runCSI(cmd, configPath, metricsBindAddress, logger) + return runCSI(cmd, configPath, logger) }, } cmd.Flags().StringVarP(&configPath, "config-file", "c", "csi.hcl", "Path to driver config (optional when using env vars)") - cmd.Flags().StringVar(&metricsBindAddress, "metrics-bind-address", ":10080", "Address to bind metrics server (set to 'off' to disable)") return cmd } -func lbControllerCmd(logger *zap.Logger) *cobra.Command { +func lbControllerCmd() *cobra.Command { var ( - apiURL string - token string - metricsBindAddress string + apiURL string + token string ) - metricsBindAddress = ":10081" cmd := &cobra.Command{ Use: "lb-controller", Short: "Run the KloudLB Service controller", @@ -98,20 +89,15 @@ func lbControllerCmd(logger *zap.Logger) *cobra.Command { ctx, cancel := signalContext(cmd.Context()) defer cancel() - if err := startMetricsServer(ctx, logger, metricsBindAddress); err != nil { - return err - } return ctrl.Run(ctx, 2) }, } cmd.Flags().StringVar(&apiURL, "api-url", "", "KKS LoadBalancer API base URL") cmd.Flags().StringVar(&token, "access-token", "", "Cluster LB access token") - cmd.Flags().StringVar(&metricsBindAddress, "metrics-bind-address", ":10081", "Address to bind metrics server (set to 'off' to disable)") return cmd } -func lbSpeakerCmd(logger *zap.Logger) *cobra.Command { - metricsBindAddress := ":10082" +func lbSpeakerCmd() *cobra.Command { cmd := &cobra.Command{ Use: "lb-speaker", Short: "Run the KloudLB L2 speaker", @@ -132,17 +118,13 @@ func lbSpeakerCmd(logger *zap.Logger) *cobra.Command { ctx, cancel := signalContext(cmd.Context()) defer cancel() - if err := startMetricsServer(ctx, logger, metricsBindAddress); err != nil { - return err - } return spk.Run(ctx) }, } - cmd.Flags().StringVar(&metricsBindAddress, "metrics-bind-address", ":10082", "Address to bind metrics server (set to 'off' to disable)") return cmd } -func runCSI(cmd *cobra.Command, configPath, metricsBindAddress string, logger *zap.Logger) error { +func runCSI(cmd *cobra.Command, configPath string, logger *zap.Logger) error { cfg, err := config.LoadClient(configPath) if err != nil { return fmt.Errorf("load config: %w", err) @@ -150,57 +132,11 @@ func runCSI(cmd *cobra.Command, configPath, metricsBindAddress string, logger *z ctx, cancel := signalContext(cmd.Context()) defer cancel() - if err := startMetricsServer(ctx, logger, metricsBindAddress); err != nil { - return err - } logger.Sugar().Infof("loaded csi driver config from %s", configPath) return driver.Run(ctx, cfg, logger) } -func startMetricsServer(ctx context.Context, logger *zap.Logger, bindAddress string) error { - bindAddress = strings.TrimSpace(bindAddress) - if bindAddress == "" || strings.EqualFold(bindAddress, "off") { - logger.Info("metrics server disabled") - return nil - } - - listener, err := net.Listen("tcp", bindAddress) - if err != nil { - return fmt.Errorf("listen metrics server on %s: %w", bindAddress, err) - } - - server := &http.Server{ - Addr: bindAddress, - Handler: metricsMux(), - ReadHeaderTimeout: 5 * time.Second, - } - - go func() { - <-ctx.Done() - shutdownCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - if err := server.Shutdown(shutdownCtx); err != nil && !errors.Is(err, http.ErrServerClosed) { - logger.Error("failed to shutdown metrics server", zap.Error(err)) - } - }() - - go func() { - logger.Sugar().Infof("metrics server listening on %s", bindAddress) - if err := server.Serve(listener); err != nil && !errors.Is(err, http.ErrServerClosed) { - logger.Error("metrics server stopped unexpectedly", zap.Error(err)) - } - }() - - return nil -} - -func metricsMux() *http.ServeMux { - mux := http.NewServeMux() - mux.Handle("/metrics", promhttp.Handler()) - return mux -} - func restConfig() (*rest.Config, error) { if cfg, err := rest.InClusterConfig(); err == nil { return cfg, nil