kloude/kks-provider-plugin
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 5 Wiki Activity

fix: use cluster metrics-server instead of plugin metrics endpoints

Browse Source
This commit is contained in:
josh
2026-07-12 15:54:27 +03:30
parent 077c472b9d
commit c5da6d9088
6 changed files with 13 additions and 115 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
+6 -8
View File
@@ -11,12 +11,12 @@ The Helm chart deploys one DaemonSet (`kks-provider-plugin-provider`) that runs
## Helm Chart
Chart path: `charts/kks-provider-plugin`
Chart URL: `https://github.com/kubelancloud/kks-provider-plugin/releases/download/v1.0.0/kks-provider-plugin-1.0.0.tgz`
Example install:
```bash
helm install kks-provider-plugin ./charts/kks-provider-plugin \
helm install kks-provider-plugin https://github.com/kubelancloud/kks-provider-plugin/releases/download/v1.0.0/kks-provider-plugin-1.0.0.tgz \
--namespace kube-system \
--create-namespace \
--set lb.serverURL=https://lb.example.kloud.team \
@@ -33,13 +33,11 @@ The container/binary entrypoint is `kks-provider` and exposes:
- `kks-provider lb-controller`
- `kks-provider lb-speaker`
Each command also starts a Prometheus metrics endpoint on `/metrics` by default:
For cluster-wide pod/node resource metrics (for `kubectl top` and autoscaling), deploy Kubernetes metrics-server once per cluster, for example:
- `kks-provider csi` on `:10080`
- `kks-provider lb-controller` on `:10081`
- `kks-provider lb-speaker` on `:10082`
You can override the bind address with `--metrics-bind-address` (or disable metrics with `--metrics-bind-address=off`).
```bash
kubectl apply -f https://github.com/kubernetes-sigs/metrics-server/releases/latest/download/high-availability-1.21+.yaml
```
## Image
charts/kks-provider-plugin/templates/provider-daemonset.yaml
-6
View File
@@ -57,8 +57,6 @@ spec:
- csi
- --config-file
- /etc/kloud-csi/driver.hcl
- --metrics-bind-address
- {{ .Values.metrics.csi.bindAddress | quote }}
env:
- name: KKS_CSI_SERVER_URL
value: {{ .Values.csi.serverURL | quote }}
@@ -157,8 +155,6 @@ spec:
command:
- /kks-provider
- lb-controller
- --metrics-bind-address
- {{ .Values.metrics.lbController.bindAddress | quote }}
env:
- name: KLOUD_LB_API_URL
value: {{ .Values.lb.serverURL | quote }}
@@ -181,8 +177,6 @@ spec:
command:
- /kks-provider
- lb-speaker
- --metrics-bind-address
- {{ .Values.metrics.lbSpeaker.bindAddress | quote }}
securityContext:
capabilities:
add:
charts/kks-provider-plugin/values.yaml
-8
View File
@@ -90,11 +90,3 @@ podAnnotations: {}
nodeSelector: {}
affinity: {}
metrics:
csi:
bindAddress: ":10080"
lbController:
bindAddress: ":10081"
lbSpeaker:
bindAddress: ":10082"
go.mod
-6
View File
@@ -5,7 +5,6 @@ go 1.24.5
require (
github.com/container-storage-interface/spec v1.9.0
github.com/hashicorp/hcl/v2 v2.24.0
github.com/prometheus/client_golang v1.22.0
github.com/spf13/cobra v1.9.1
github.com/vishvananda/netlink v1.3.1
go.uber.org/zap v1.27.0
@@ -21,8 +20,6 @@ require (
require (
github.com/agext/levenshtein v1.2.1 // indirect
github.com/apparentlymart/go-textseg/v15 v15.0.0 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/cespare/xxhash/v2 v2.3.0 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/emicklei/go-restful/v3 v3.11.0 // indirect
github.com/fxamacker/cbor/v2 v2.7.0 // indirect
@@ -45,9 +42,6 @@ require (
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/prometheus/client_model v0.6.1 // indirect
github.com/prometheus/common v0.62.0 // indirect
github.com/prometheus/procfs v0.15.1 // indirect
github.com/spf13/pflag v1.0.6 // indirect
github.com/vishvananda/netns v0.0.5 // indirect
github.com/x448/float16 v0.8.4 // indirect
go.sum
-16
View File
@@ -2,10 +2,6 @@ github.com/agext/levenshtein v1.2.1 h1:QmvMAjj2aEICytGiWzmxoE0x2KZvE0fvmqMOfy2tj
github.com/agext/levenshtein v1.2.1/go.mod h1:JEDfjyjHDjOF/1e4FlBE/PkbqA9OfWu2ki2W0IB5558=
github.com/apparentlymart/go-textseg/v15 v15.0.0 h1:uYvfpb3DyLSCGWnctWKGj857c6ew1u1fNQOlOtuGxQY=
github.com/apparentlymart/go-textseg/v15 v15.0.0/go.mod h1:K8XmNZdhEBkdlyDdvbmmsvpAG721bKi0joRfFdHIWJ4=
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/container-storage-interface/spec v1.9.0 h1:zKtX4STsq31Knz3gciCYCi1SXtO2HJDecIjDVboYavY=
github.com/container-storage-interface/spec v1.9.0/go.mod h1:ZfDu+3ZRyeVqxZM0Ds19MVLkN2d1XJ5MAfi1L3VjlT0=
github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
@@ -55,8 +51,6 @@ github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnr
github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo=
github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ=
github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
@@ -64,8 +58,6 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
github.com/mitchellh/go-wordwrap v1.0.1 h1:TLuKupo69TCn6TQSyGxwI1EblZZEsQ0vMlAFQflz0v0=
@@ -87,14 +79,6 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/prometheus/client_golang v1.22.0 h1:rb93p9lokFEsctTys46VnV1kLCDpVZ0a/Y92Vm0Zc6Q=
github.com/prometheus/client_golang v1.22.0/go.mod h1:R7ljNsLXhuQXYZYtw6GAE9AZg8Y7vEW5scdCXrWRXC0=
github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E=
github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY=
github.com/prometheus/common v0.62.0 h1:xasJaQlnWAeyHdUBeGjXmutelfJHWMRr+Fg4QszZ2Io=
github.com/prometheus/common v0.62.0/go.mod h1:vyBcEuLSvWos9B1+CyL7JZ2up+uFzXhkqml0W5zIY1I=
github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc=
github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk=
github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII=
github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o=
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
main.go
+5 -69
View File
@@ -2,13 +2,9 @@ package main
import (
"context"
"errors"
"fmt"
"net"
"net/http"
"os"
"os/signal"
"strings"
"syscall"
"time"
@@ -17,7 +13,6 @@ import (
"github.com/KubelanCloud/kks-provider-plugin/pkg/kloudlb/controller"
"github.com/KubelanCloud/kks-provider-plugin/pkg/kloudlb/speaker"
lbapi "github.com/KubelanCloud/kks-provider-plugin/pkg/lb/api"
"github.com/prometheus/client_golang/prometheus/promhttp"
"github.com/spf13/cobra"
"go.uber.org/zap"
"k8s.io/client-go/kubernetes"
@@ -39,7 +34,7 @@ func main() {
Long: "Runs the in-cluster provider components for CSI and LoadBalancer integration with kks management services.",
}
rootCmd.AddCommand(csiCmd(logger), lbControllerCmd(logger), lbSpeakerCmd(logger))
rootCmd.AddCommand(csiCmd(logger), lbControllerCmd(), lbSpeakerCmd())
if err := rootCmd.Execute(); err != nil {
os.Exit(1)
@@ -48,26 +43,22 @@ func main() {
func csiCmd(logger *zap.Logger) *cobra.Command {
configPath := "csi.hcl"
metricsBindAddress := ":10080"
cmd := &cobra.Command{
Use: "csi",
Short: "Run the Kloud CSI driver",
RunE: func(cmd *cobra.Command, args []string) error {
return runCSI(cmd, configPath, metricsBindAddress, logger)
return runCSI(cmd, configPath, logger)
},
}
cmd.Flags().StringVarP(&configPath, "config-file", "c", "csi.hcl", "Path to driver config (optional when using env vars)")
cmd.Flags().StringVar(&metricsBindAddress, "metrics-bind-address", ":10080", "Address to bind metrics server (set to 'off' to disable)")
return cmd
}
func lbControllerCmd(logger *zap.Logger) *cobra.Command {
func lbControllerCmd() *cobra.Command {
var (
apiURL string
token string
metricsBindAddress string
)
metricsBindAddress = ":10081"
cmd := &cobra.Command{
Use: "lb-controller",
Short: "Run the KloudLB Service controller",
@@ -98,20 +89,15 @@ func lbControllerCmd(logger *zap.Logger) *cobra.Command {
ctx, cancel := signalContext(cmd.Context())
defer cancel()
if err := startMetricsServer(ctx, logger, metricsBindAddress); err != nil {
return err
}
return ctrl.Run(ctx, 2)
},
}
cmd.Flags().StringVar(&apiURL, "api-url", "", "KKS LoadBalancer API base URL")
cmd.Flags().StringVar(&token, "access-token", "", "Cluster LB access token")
cmd.Flags().StringVar(&metricsBindAddress, "metrics-bind-address", ":10081", "Address to bind metrics server (set to 'off' to disable)")
return cmd
}
func lbSpeakerCmd(logger *zap.Logger) *cobra.Command {
metricsBindAddress := ":10082"
func lbSpeakerCmd() *cobra.Command {
cmd := &cobra.Command{
Use: "lb-speaker",
Short: "Run the KloudLB L2 speaker",
@@ -132,17 +118,13 @@ func lbSpeakerCmd(logger *zap.Logger) *cobra.Command {
ctx, cancel := signalContext(cmd.Context())
defer cancel()
if err := startMetricsServer(ctx, logger, metricsBindAddress); err != nil {
return err
}
return spk.Run(ctx)
},
}
cmd.Flags().StringVar(&metricsBindAddress, "metrics-bind-address", ":10082", "Address to bind metrics server (set to 'off' to disable)")
return cmd
}
func runCSI(cmd *cobra.Command, configPath, metricsBindAddress string, logger *zap.Logger) error {
func runCSI(cmd *cobra.Command, configPath string, logger *zap.Logger) error {
cfg, err := config.LoadClient(configPath)
if err != nil {
return fmt.Errorf("load config: %w", err)
@@ -150,57 +132,11 @@ func runCSI(cmd *cobra.Command, configPath, metricsBindAddress string, logger *z
ctx, cancel := signalContext(cmd.Context())
defer cancel()
if err := startMetricsServer(ctx, logger, metricsBindAddress); err != nil {
return err
}
logger.Sugar().Infof("loaded csi driver config from %s", configPath)
return driver.Run(ctx, cfg, logger)
}
func startMetricsServer(ctx context.Context, logger *zap.Logger, bindAddress string) error {
bindAddress = strings.TrimSpace(bindAddress)
if bindAddress == "" || strings.EqualFold(bindAddress, "off") {
logger.Info("metrics server disabled")
return nil
}
listener, err := net.Listen("tcp", bindAddress)
if err != nil {
return fmt.Errorf("listen metrics server on %s: %w", bindAddress, err)
}
server := &http.Server{
Addr: bindAddress,
Handler: metricsMux(),
ReadHeaderTimeout: 5 * time.Second,
}
go func() {
<-ctx.Done()
shutdownCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
if err := server.Shutdown(shutdownCtx); err != nil && !errors.Is(err, http.ErrServerClosed) {
logger.Error("failed to shutdown metrics server", zap.Error(err))
}
}()
go func() {
logger.Sugar().Infof("metrics server listening on %s", bindAddress)
if err := server.Serve(listener); err != nil && !errors.Is(err, http.ErrServerClosed) {
logger.Error("metrics server stopped unexpectedly", zap.Error(err))
}
}()
return nil
}
func metricsMux() *http.ServeMux {
mux := http.NewServeMux()
mux.Handle("/metrics", promhttp.Handler())
return mux
}
func restConfig() (*rest.Config, error) {
if cfg, err := rest.InClusterConfig(); err == nil {
return cfg, nil