From 077c472b9d00fee16e1dcd72cf88a8faa7dc40ee Mon Sep 17 00:00:00 2001 From: Josh Date: Sun, 12 Jul 2026 14:42:22 +0330 Subject: [PATCH] feat: consolidate provider plugin and chart --- .cr.yaml | 2 +- .github/workflows/build.yaml | 6 +- .github/workflows/release-chart.yaml | 2 +- .github/workflows/release.yaml | 8 +- Dockerfile | 6 +- README.md | 285 ++------------- charts/kks-provider-plugin/Chart.yaml | 18 + .../templates/_helpers.tpl | 73 ++++ .../templates/configmap.yaml | 4 +- .../templates/csidriver.yaml | 4 +- .../templates/lb-clusterrole.yaml | 18 + .../templates/lb-clusterrolebinding.yaml | 16 + .../templates/lb-serviceaccount.yaml | 9 + .../templates/provider-daemonset.yaml | 220 +++++++++++ .../templates/rbac-controller.yaml | 21 +- .../templates/rbac-node.yaml | 27 ++ .../kks-provider-plugin/templates/secret.yaml | 28 ++ .../templates/storageclass.yaml | 49 +++ charts/kks-provider-plugin/values.yaml | 100 +++++ charts/kloud-csi/Chart.yaml | 16 - charts/kloud-csi/templates/_helpers.tpl | 77 ---- .../templates/controller-deployment.yaml | 108 ------ .../kloud-csi/templates/node-daemonset.yaml | 136 ------- charts/kloud-csi/templates/rbac-node.yaml | 36 -- charts/kloud-csi/templates/secret.yaml | 13 - charts/kloud-csi/templates/storageclass.yaml | 49 --- charts/kloud-csi/values.yaml | 87 ----- examples/csi-client.hcl | 4 +- go.mod | 51 ++- go.sum | 166 ++++++++- main.go | 192 +++++++++- pkg/csi/api/client.go | 2 +- pkg/csi/api/client_test.go | 2 +- pkg/csi/client/backend.go | 6 +- pkg/csi/driver/controller.go | 2 +- pkg/csi/driver/controller_test.go | 2 +- pkg/csi/driver/driver.go | 6 +- pkg/csi/driver/identity.go | 2 +- pkg/csi/driver/node.go | 2 +- pkg/csi/driver/volume.go | 2 +- pkg/kloudlb/constants.go | 9 + pkg/kloudlb/controller/controller.go | 277 ++++++++++++++ pkg/kloudlb/controller/controller_test.go | 40 ++ pkg/kloudlb/speaker/speaker.go | 345 ++++++++++++++++++ pkg/lb/api/client.go | 127 +++++++ pkg/lb/api/client_test.go | 39 ++ pkg/lb/api/errors.go | 17 + pkg/lb/provisioner/types.go | 13 + 48 files changed, 1870 insertions(+), 854 deletions(-) create mode 100644 charts/kks-provider-plugin/Chart.yaml create mode 100644 charts/kks-provider-plugin/templates/_helpers.tpl rename charts/{kloud-csi => kks-provider-plugin}/templates/configmap.yaml (52%) rename charts/{kloud-csi => kks-provider-plugin}/templates/csidriver.yaml (61%) create mode 100644 charts/kks-provider-plugin/templates/lb-clusterrole.yaml create mode 100644 charts/kks-provider-plugin/templates/lb-clusterrolebinding.yaml create mode 100644 charts/kks-provider-plugin/templates/lb-serviceaccount.yaml create mode 100644 charts/kks-provider-plugin/templates/provider-daemonset.yaml rename charts/{kloud-csi => kks-provider-plugin}/templates/rbac-controller.yaml (70%) create mode 100644 charts/kks-provider-plugin/templates/rbac-node.yaml create mode 100644 charts/kks-provider-plugin/templates/secret.yaml create mode 100644 charts/kks-provider-plugin/templates/storageclass.yaml create mode 100644 charts/kks-provider-plugin/values.yaml delete mode 100644 charts/kloud-csi/Chart.yaml delete mode 100644 charts/kloud-csi/templates/_helpers.tpl delete mode 100644 charts/kloud-csi/templates/controller-deployment.yaml delete mode 100644 charts/kloud-csi/templates/node-daemonset.yaml delete mode 100644 charts/kloud-csi/templates/rbac-node.yaml delete mode 100644 charts/kloud-csi/templates/secret.yaml delete mode 100644 charts/kloud-csi/templates/storageclass.yaml delete mode 100644 charts/kloud-csi/values.yaml create mode 100644 pkg/kloudlb/constants.go create mode 100644 pkg/kloudlb/controller/controller.go create mode 100644 pkg/kloudlb/controller/controller_test.go create mode 100644 pkg/kloudlb/speaker/speaker.go create mode 100644 pkg/lb/api/client.go create mode 100644 pkg/lb/api/client_test.go create mode 100644 pkg/lb/api/errors.go create mode 100644 pkg/lb/provisioner/types.go diff --git a/.cr.yaml b/.cr.yaml index 3a57c9b..67aac3e 100644 --- a/.cr.yaml +++ b/.cr.yaml @@ -1,4 +1,4 @@ owner: KubelanCloud -git-repo: kks-csi-plugin +git-repo: kks-provider-plugin pages-branch: gh-pages pages-index-path: index.yaml diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index ec09fba..dcee67d 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -21,8 +21,8 @@ jobs: - name: Container image name (GHCR is lowercase) run: | owner=$(echo "${GITHUB_REPOSITORY_OWNER}" | tr '[:upper:]' '[:lower:]') - echo "IMAGE_NAME=${REGISTRY_HOST}/${owner}/kks-csi-plugin" >> "$GITHUB_ENV" - echo "APP_VERSION=$(grep '^appVersion:' charts/kloud-csi/Chart.yaml | awk '{print $2}' | tr -d '\"')" >> "$GITHUB_ENV" + echo "IMAGE_NAME=${REGISTRY_HOST}/${owner}/kks-provider-plugin" >> "$GITHUB_ENV" + echo "APP_VERSION=$(grep '^appVersion:' charts/kks-provider-plugin/Chart.yaml | awk '{print $2}' | tr -d '\"')" >> "$GITHUB_ENV" - uses: docker/setup-buildx-action@v3 @@ -61,4 +61,4 @@ jobs: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | owner=$(echo "${GITHUB_REPOSITORY_OWNER}" | tr '[:upper:]' '[:lower:]') - gh api --method PATCH "/orgs/${owner}/packages/container/kks-csi-plugin/visibility" -f visibility=public || true + gh api --method PATCH "/orgs/${owner}/packages/container/kks-provider-plugin/visibility" -f visibility=public || true diff --git a/.github/workflows/release-chart.yaml b/.github/workflows/release-chart.yaml index a72f5b6..4c9599e 100644 --- a/.github/workflows/release-chart.yaml +++ b/.github/workflows/release-chart.yaml @@ -65,4 +65,4 @@ jobs: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | owner=$(echo "${GITHUB_REPOSITORY_OWNER}" | tr '[:upper:]' '[:lower:]') - gh api --method PATCH "/orgs/${owner}/packages/container/charts%2Fkloud-csi/visibility" -f visibility=public || true + gh api --method PATCH "/orgs/${owner}/packages/container/charts%2Fkks-provider-plugin/visibility" -f visibility=public || true diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 9cf510d..1c81ab8 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -4,7 +4,7 @@ on: push: tags: - "v*" - - "kloud-csi-*" + - "kks-provider-plugin-*" permissions: contents: write @@ -38,7 +38,7 @@ jobs: if: steps.version.outputs.is_app_release == 'true' run: | version="${{ steps.version.outputs.app_version }}" - chart_dir="charts/kloud-csi" + chart_dir="charts/kks-provider-plugin" sed -i "s/^version:.*/version: ${version}/" "${chart_dir}/Chart.yaml" sed -i "s/^appVersion:.*/appVersion: \"${version}\"/" "${chart_dir}/Chart.yaml" @@ -46,7 +46,7 @@ jobs: helm lint "${chart_dir}" mkdir -p dist helm package "${chart_dir}" --destination dist - echo "CHART_PACKAGE=dist/kloud-csi-${version}.tgz" >> "$GITHUB_ENV" + echo "CHART_PACKAGE=dist/kks-provider-plugin-${version}.tgz" >> "$GITHUB_ENV" - name: Build changelog id: changelog @@ -112,4 +112,4 @@ jobs: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | owner=$(echo "${GITHUB_REPOSITORY_OWNER}" | tr '[:upper:]' '[:lower:]') - gh api --method PATCH "/orgs/${owner}/packages/container/charts%2Fkloud-csi/visibility" -f visibility=public || true + gh api --method PATCH "/orgs/${owner}/packages/container/charts%2Fkks-provider-plugin/visibility" -f visibility=public || true diff --git a/Dockerfile b/Dockerfile index 407cb23..73deced 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,10 +5,10 @@ RUN apk add --no-cache git COPY go.mod go.sum ./ RUN go mod download COPY . . -RUN CGO_ENABLED=0 go build -o /kks-csi . +RUN CGO_ENABLED=0 go build -o /kks-provider . FROM alpine:3.21 RUN apk add --no-cache mount util-linux e2fsprogs findmnt -COPY --from=build /kks-csi /kks-csi +COPY --from=build /kks-provider /kks-provider USER 0:0 -ENTRYPOINT ["/kks-csi"] +ENTRYPOINT ["/kks-provider"] diff --git a/README.md b/README.md index 46800b8..db04380 100644 --- a/README.md +++ b/README.md @@ -1,279 +1,48 @@ -# kks-csi-plugin +# kks-provider-plugin -Kloud CSI driver for Kubernetes user clusters. This plugin runs inside each user cluster and implements the [Container Storage Interface (CSI)](https://github.com/container-storage-interface/spec) so workloads can use persistent volumes backed by Kloud storage. +Combined provider plugin for Kubernetes user clusters. -Storage operations are forwarded to the **kks management CSI server** over HTTP. The driver does not talk to storage hardware directly; it translates Kubernetes CSI calls into REST API requests against the management plane. +This repository now contains a single plugin and Helm chart that bundles: -## Architecture +- KloudLB controller + speaker logic +- Kloud CSI controller + node logic -``` -┌─────────────────────────────────────────────────────────────┐ -│ User Kubernetes cluster │ -│ │ -│ ┌──────────────────┐ ┌─────────────────────────────┐ │ -│ │ CSI sidecars │ │ kks-csi-plugin │ │ -│ │ (provisioner, │────▶│ controller / node modes │ │ -│ │ attacher, etc.) │ └──────────────┬──────────────┘ │ -│ └──────────────────┘ │ HTTP │ -│ ▼ │ -└────────────────────────────────────────────┼─────────────────┘ - │ - ▼ - ┌──────────────────────────────┐ - │ kks management CSI server │ - │ (Kloud control plane) │ - └──────────────────────────────┘ -``` +The Helm chart deploys one DaemonSet (`kks-provider-plugin-provider`) that runs all required LB and CSI containers in the same pod on each node. -The Helm chart deploys two workloads: +## Helm Chart -| Component | Kind | Role | -|-----------|------|------| -| **Controller** | Deployment | Handles volume create/delete and publish/unpublish via CSI controller RPCs | -| **Node** | DaemonSet | Stages and publishes volumes on each node (format, mount, bind-mount) | +Chart path: `charts/kks-provider-plugin` -Standard CSI sidecars are bundled with each workload: - -- **csi-provisioner** and **csi-attacher** on the controller -- **csi-node-driver-registrar** on each node -- **liveness-probe** on both - -## Features - -- Dynamic provisioning via two `StorageClass` types (optional, enabled by default) -- Controller publish/unpublish (attach/detach) -- Node stage/unstage and publish/unpublish (mount operations) -- Block volumes are not supported -- Volume expansion is not supported - -Driver name: `storage.csi.kloud.team` - -## Prerequisites - -- Kubernetes **1.28+** -- Network access from the user cluster to the kks management CSI server -- A cluster **CSI access token** and **server URL** from your Kloud cluster details - -## Install with Helm - -### From the published chart repository - -Charts are published as [GitHub Releases](https://github.com/KubelanCloud/kks-csi-plugin/releases) and to GHCR as OCI artifacts. - -**GitHub Release (public):** +Example install: ```bash -helm install kloud-csi \ - https://github.com/KubelanCloud/kks-csi-plugin/releases/download/kloud-csi-0.1.0/kloud-csi-0.1.0.tgz \ +helm install kks-provider-plugin ./charts/kks-provider-plugin \ --namespace kube-system \ --create-namespace \ - --set serverURL=https://csi.example.kloud.team \ - --set accessToken="YOUR_CLUSTER_CSI_ACCESS_TOKEN" + --set lb.serverURL=https://lb.example.kloud.team \ + --set lb.accessToken="$LB_TOKEN" \ + --set csi.serverURL=https://csi.example.kloud.team \ + --set csi.accessToken="$CSI_TOKEN" ``` -**OCI (requires GHCR access):** +## Binary Commands -```bash -helm registry login ghcr.io +The container/binary entrypoint is `kks-provider` and exposes: -helm install kloud-csi oci://ghcr.io/kubelancloud/charts/kloud-csi \ - --version 0.1.0 \ - --namespace kube-system \ - --create-namespace \ - --set serverURL=https://csi.example.kloud.team \ - --set accessToken="YOUR_CLUSTER_CSI_ACCESS_TOKEN" -``` +- `kks-provider csi` +- `kks-provider lb-controller` +- `kks-provider lb-speaker` -To allow anonymous OCI pulls, set the `charts/kloud-csi` package visibility to public under **GitHub → Packages**. +Each command also starts a Prometheus metrics endpoint on `/metrics` by default: -### From a local checkout +- `kks-provider csi` on `:10080` +- `kks-provider lb-controller` on `:10081` +- `kks-provider lb-speaker` on `:10082` -```bash -helm install kloud-csi ./charts/kloud-csi \ - --namespace kube-system \ - --create-namespace \ - --set serverURL=https://csi.example.kloud.team \ - --set accessToken="YOUR_CLUSTER_CSI_ACCESS_TOKEN" -``` +You can override the bind address with `--metrics-bind-address` (or disable metrics with `--metrics-bind-address=off`). -### Using an existing secret +## Image -If you already have a secret containing the access token: +Default image repository is: -```bash -kubectl create secret generic kloud-csi-credentials \ - --namespace kube-system \ - --from-literal=access-token="YOUR_CLUSTER_CSI_ACCESS_TOKEN" - -helm install kloud-csi ./charts/kloud-csi \ - --namespace kube-system \ - --set serverURL=https://csi.example.kloud.team \ - --set existingSecret=kloud-csi-credentials -``` - -### Verify the install - -```bash -kubectl get pods -n kube-system -l app.kubernetes.io/name=kloud-csi -kubectl get csidriver storage.csi.kloud.team -kubectl get storageclass kloud-csi -``` - -## Helm chart - -Chart path: [`charts/kloud-csi`](charts/kloud-csi) - -Published releases are available at [GitHub Releases](https://github.com/KubelanCloud/kks-csi-plugin/releases) and as OCI charts at `oci://ghcr.io/kubelancloud/charts/kloud-csi`. Bump `version` in [`Chart.yaml`](charts/kloud-csi/Chart.yaml) to publish a new chart release. - -### Required values - -| Value | Description | -|-------|-------------| -| `serverURL` | Base URL of the kks management CSI server | -| `accessToken` | Cluster CSI access token (required unless `existingSecret` is set) | - -Helm fails at render time if `serverURL` or credentials are missing. - -### Common values - -| Value | Default | Description | -|-------|---------|-------------| -| `image.repository` | `ghcr.io/kubelancloud/kloud-csi-plugin` | Driver container image | -| `image.tag` | *(chart appVersion)* | Image tag (defaults to chart `appVersion` when empty) | -| `imagePullSecrets` | `[]` | Pull secrets for private registries such as GHCR | -| `existingSecret` | `""` | Use an existing secret instead of creating one | -| `existingSecretAccessTokenKey` | `access-token` | Key in the secret holding the token | -| `driver.name` | `storage.csi.kloud.team` | CSI driver name | -| `storageClass.enabled` | `true` | Create StorageClasses | -| `storageClass.immediate.enabled` | `true` | Create immediate-binding StorageClass | -| `storageClass.immediate.name` | `kloud-csi` | Immediate-binding StorageClass name | -| `storageClass.immediate.isDefault` | `true` | Mark immediate StorageClass as default | -| `storageClass.waitForFirstConsumer.enabled` | `true` | Create WaitForFirstConsumer StorageClass | -| `storageClass.waitForFirstConsumer.name` | `kloud-csi-wait-for-first-consumer` | WaitForFirstConsumer StorageClass name | -| `storageClass.waitForFirstConsumer.isDefault` | `false` | Mark WaitForFirstConsumer StorageClass as default | -| `storageClass.reclaimPolicy` | `Delete` | `Delete` or `Retain` | -| `controller.replicas` | `1` | Controller deployment replicas | -| `rbac.create` | `true` | Create RBAC for controller and node | - -See [`charts/kloud-csi/values.yaml`](charts/kloud-csi/values.yaml) for the full list, including sidecar image versions and resource limits. - -### Chart resources - -The chart creates: - -- `CSIDriver` — registers the driver with Kubernetes -- `StorageClass` (Immediate) — optional, default class for immediate provisioning -- `StorageClass` (WaitForFirstConsumer) — optional, delayed provisioning until pod scheduling -- `Deployment` — controller + sidecars -- `DaemonSet` — node plugin + registrar on every node -- `ConfigMap` — minimal HCL stub (settings come from env vars) -- `Secret` — access token (unless `existingSecret` is used) -- RBAC — service accounts and roles for controller and node - -### Upgrade and uninstall - -```bash -helm upgrade kloud-csi ./charts/kloud-csi \ - --namespace kube-system \ - --reuse-values \ - --set serverURL=https://csi.example.kloud.team - -helm uninstall kloud-csi --namespace kube-system -``` - -## Using persistent volumes - -With the default immediate StorageClass installed, create a PVC: - -```yaml -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: my-data -spec: - accessModes: - - ReadWriteOnce - storageClassName: kloud-csi - resources: - requests: - storage: 10Gi -``` - -Then mount it in a pod as usual. - -To delay provisioning until the first pod is scheduled, use the second class: - -```yaml -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: my-data-wait -spec: - accessModes: - - ReadWriteOnce - storageClassName: kloud-csi-wait-for-first-consumer - resources: - requests: - storage: 10Gi -``` - -## Configuration reference - -The driver accepts configuration from an HCL file, environment variables, or both. Environment variables take precedence and are what the Helm chart uses. - -### HCL file - -Example: [`examples/csi-client.hcl`](examples/csi-client.hcl) - -```hcl -driver { - name = "storage.csi.kloud.team" - endpoint = "unix:///var/lib/kubelet/plugins/storage.csi.kloud.team/csi.sock" - mode = "all" # all | controller | node -} - -client { - server_url = "https://csi.example.kloud.team" - access_token = "YOUR_CLUSTER_CSI_ACCESS_TOKEN" -} -``` - -### Environment variables - -| Variable | Description | -|----------|-------------| -| `KKS_CSI_SERVER_URL` | Management CSI server base URL | -| `KKS_CSI_ACCESS_TOKEN` | Cluster access token | -| `KKS_CSI_DRIVER_MODE` | `controller`, `node`, or `all` | -| `KKS_CSI_NODE_ID` | Node identifier (set automatically on node pods) | -| `KKS_CSI_DRIVER_NAME` | CSI driver name override | -| `KKS_CSI_DRIVER_ENDPOINT` | gRPC socket path | -| `KKS_CSI_CLIENT_TIMEOUT_SECONDS` | HTTP client timeout (default: 30) | - -## Standalone / development - -Build and run locally: - -```bash -go build -o kks-csi . -./kks-csi -c examples/csi-client.hcl -``` - -Or with Docker: - -```bash -docker build -t kks-csi . -docker run --rm -v "$(pwd)/examples/csi-client.hcl:/csi.hcl:ro" kks-csi -c /csi.hcl -``` - -Container images are published to `ghcr.io/kubelancloud/kloud-csi-plugin` on pushes to `main`. - -## Development - -```bash -go test ./... -``` - -## License - -See repository license terms. +`ghcr.io/kubelancloud/kks-provider-plugin` diff --git a/charts/kks-provider-plugin/Chart.yaml b/charts/kks-provider-plugin/Chart.yaml new file mode 100644 index 0000000..d26e091 --- /dev/null +++ b/charts/kks-provider-plugin/Chart.yaml @@ -0,0 +1,18 @@ +apiVersion: v2 +name: kks-provider-plugin +description: Combined Kloud provider plugin chart (LoadBalancer + CSI) for kks clusters +type: application +version: 0.1.0 +appVersion: "0.1.0" +kubeVersion: ">=1.28.0-0" +home: https://github.com/KubelanCloud/kks-provider-plugin +sources: + - https://github.com/KubelanCloud/kks-provider-plugin +keywords: + - provider + - loadbalancer + - csi + - storage + - kloud +maintainers: + - name: Kloud Team diff --git a/charts/kks-provider-plugin/templates/_helpers.tpl b/charts/kks-provider-plugin/templates/_helpers.tpl new file mode 100644 index 0000000..b87150b --- /dev/null +++ b/charts/kks-provider-plugin/templates/_helpers.tpl @@ -0,0 +1,73 @@ +{{- define "kks-provider-plugin.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{- define "kks-provider-plugin.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{- define "kks-provider-plugin.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{- define "kks-provider-plugin.labels" -}} +helm.sh/chart: {{ include "kks-provider-plugin.chart" . }} +{{ include "kks-provider-plugin.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{- define "kks-provider-plugin.selectorLabels" -}} +app.kubernetes.io/name: {{ include "kks-provider-plugin.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{- define "kks-provider-plugin.image" -}} +{{- printf "%s:%s" .Values.image.repository (default .Chart.AppVersion .Values.image.tag) }} +{{- end }} + +{{- define "kks-provider-plugin.daemonSetName" -}} +{{- printf "%s-provider" (include "kks-provider-plugin.fullname" .) }} +{{- end }} + +{{- define "kks-provider-plugin.lbServiceAccountName" -}} +{{- if .Values.lb.serviceAccount.create }} +{{- default (printf "%s-lb-sa" (include "kks-provider-plugin.fullname" .)) .Values.lb.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.lb.serviceAccount.name }} +{{- end }} +{{- end }} + +{{- define "kks-provider-plugin.lbSecretName" -}} +{{- default (printf "%s-lb-config" (include "kks-provider-plugin.fullname" .)) .Values.lb.existingSecret }} +{{- end }} + +{{- define "kks-provider-plugin.csiSecretName" -}} +{{- default (printf "%s-csi-config" (include "kks-provider-plugin.fullname" .)) .Values.csi.existingSecret }} +{{- end }} + +{{- define "kks-provider-plugin.validateRequired" -}} +{{- if and (not .Values.lb.existingSecret) (not .Values.lb.accessToken) }} +{{- fail "lb.accessToken or lb.existingSecret is required" }} +{{- end }} +{{- if not .Values.lb.serverURL }} +{{- fail "lb.serverURL is required" }} +{{- end }} +{{- if and (not .Values.csi.existingSecret) (not .Values.csi.accessToken) }} +{{- fail "csi.accessToken or csi.existingSecret is required" }} +{{- end }} +{{- if not .Values.csi.serverURL }} +{{- fail "csi.serverURL is required" }} +{{- end }} +{{- end }} diff --git a/charts/kloud-csi/templates/configmap.yaml b/charts/kks-provider-plugin/templates/configmap.yaml similarity index 52% rename from charts/kloud-csi/templates/configmap.yaml rename to charts/kks-provider-plugin/templates/configmap.yaml index 0d74f80..9f20842 100644 --- a/charts/kloud-csi/templates/configmap.yaml +++ b/charts/kks-provider-plugin/templates/configmap.yaml @@ -1,10 +1,10 @@ apiVersion: v1 kind: ConfigMap metadata: - name: {{ include "kloud-csi.fullname" . }}-driver + name: {{ include "kks-provider-plugin.fullname" . }}-driver namespace: {{ .Release.Namespace }} labels: - {{- include "kloud-csi.labels" . | nindent 4 }} + {{- include "kks-provider-plugin.labels" . | nindent 4 }} data: driver.hcl: | driver {} diff --git a/charts/kloud-csi/templates/csidriver.yaml b/charts/kks-provider-plugin/templates/csidriver.yaml similarity index 61% rename from charts/kloud-csi/templates/csidriver.yaml rename to charts/kks-provider-plugin/templates/csidriver.yaml index be7dd69..54dc89b 100644 --- a/charts/kloud-csi/templates/csidriver.yaml +++ b/charts/kks-provider-plugin/templates/csidriver.yaml @@ -1,9 +1,9 @@ apiVersion: storage.k8s.io/v1 kind: CSIDriver metadata: - name: {{ .Values.driver.name }} + name: {{ .Values.csi.driver.name }} labels: - {{- include "kloud-csi.labels" . | nindent 4 }} + {{- include "kks-provider-plugin.labels" . | nindent 4 }} spec: attachRequired: true podInfoOnMount: false diff --git a/charts/kks-provider-plugin/templates/lb-clusterrole.yaml b/charts/kks-provider-plugin/templates/lb-clusterrole.yaml new file mode 100644 index 0000000..f294925 --- /dev/null +++ b/charts/kks-provider-plugin/templates/lb-clusterrole.yaml @@ -0,0 +1,18 @@ +{{- if .Values.rbac.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "kks-provider-plugin.fullname" . }}-lb + labels: + {{- include "kks-provider-plugin.labels" . | nindent 4 }} +rules: + - apiGroups: [""] + resources: ["services"] + verbs: ["get", "list", "watch", "update", "patch"] + - apiGroups: [""] + resources: ["services/status"] + verbs: ["get", "update", "patch"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "list", "watch", "create", "update", "patch"] +{{- end }} diff --git a/charts/kks-provider-plugin/templates/lb-clusterrolebinding.yaml b/charts/kks-provider-plugin/templates/lb-clusterrolebinding.yaml new file mode 100644 index 0000000..85a96b7 --- /dev/null +++ b/charts/kks-provider-plugin/templates/lb-clusterrolebinding.yaml @@ -0,0 +1,16 @@ +{{- if .Values.rbac.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "kks-provider-plugin.fullname" . }}-lb + labels: + {{- include "kks-provider-plugin.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ include "kks-provider-plugin.fullname" . }}-lb +subjects: + - kind: ServiceAccount + name: {{ include "kks-provider-plugin.lbServiceAccountName" . }} + namespace: {{ .Release.Namespace }} +{{- end }} diff --git a/charts/kks-provider-plugin/templates/lb-serviceaccount.yaml b/charts/kks-provider-plugin/templates/lb-serviceaccount.yaml new file mode 100644 index 0000000..f0c4e4c --- /dev/null +++ b/charts/kks-provider-plugin/templates/lb-serviceaccount.yaml @@ -0,0 +1,9 @@ +{{- if .Values.lb.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "kks-provider-plugin.lbServiceAccountName" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "kks-provider-plugin.labels" . | nindent 4 }} +{{- end }} diff --git a/charts/kks-provider-plugin/templates/provider-daemonset.yaml b/charts/kks-provider-plugin/templates/provider-daemonset.yaml new file mode 100644 index 0000000..7aabb45 --- /dev/null +++ b/charts/kks-provider-plugin/templates/provider-daemonset.yaml @@ -0,0 +1,220 @@ +{{- include "kks-provider-plugin.validateRequired" . }} +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: {{ include "kks-provider-plugin.daemonSetName" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "kks-provider-plugin.labels" . | nindent 4 }} + app.kubernetes.io/component: provider +spec: + selector: + matchLabels: + {{- include "kks-provider-plugin.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: provider + template: + metadata: + labels: + {{- include "kks-provider-plugin.selectorLabels" . | nindent 8 }} + app.kubernetes.io/component: provider + {{- with .Values.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + serviceAccountName: {{ include "kks-provider-plugin.lbServiceAccountName" . }} + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + priorityClassName: {{ .Values.csi.node.priorityClassName }} + hostNetwork: true + hostPID: true + {{- with .Values.csi.node.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + containers: + - name: csi-driver + image: {{ include "kks-provider-plugin.image" . }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + securityContext: + privileged: true + command: + - /kks-provider + args: + - csi + - --config-file + - /etc/kloud-csi/driver.hcl + - --metrics-bind-address + - {{ .Values.metrics.csi.bindAddress | quote }} + env: + - name: KKS_CSI_SERVER_URL + value: {{ .Values.csi.serverURL | quote }} + - name: KKS_CSI_ACCESS_TOKEN + valueFrom: + secretKeyRef: + name: {{ include "kks-provider-plugin.csiSecretName" . }} + key: {{ .Values.csi.existingSecretAccessTokenKey }} + - name: KKS_CSI_DRIVER_MODE + value: all + - name: KKS_CSI_DRIVER_ENDPOINT + value: unix:///csi/csi.sock + - name: KKS_CSI_DRIVER_NAME + value: {{ .Values.csi.driver.name | quote }} + - name: KKS_CSI_NODE_ID + valueFrom: + fieldRef: + fieldPath: spec.nodeName + volumeMounts: + - name: plugin-dir + mountPath: /csi + - name: kubelet-dir + mountPath: /var/lib/kubelet + mountPropagation: Bidirectional + - name: device-dir + mountPath: /dev + - name: config + mountPath: /etc/kloud-csi + {{- with .Values.csi.node.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + - name: csi-provisioner + image: {{ .Values.csi.sidecars.provisioner.repository }}:{{ .Values.csi.sidecars.provisioner.tag }} + args: + - --csi-address=/csi/csi.sock + - --v=2 + - --feature-gates=Topology=false + - --timeout=60s + - --leader-election + - --default-fstype=ext4 + volumeMounts: + - name: plugin-dir + mountPath: /csi + {{- with .Values.csi.controller.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + - name: csi-attacher + image: {{ .Values.csi.sidecars.attacher.repository }}:{{ .Values.csi.sidecars.attacher.tag }} + args: + - --csi-address=/csi/csi.sock + - --v=2 + - --leader-election + volumeMounts: + - name: plugin-dir + mountPath: /csi + {{- with .Values.csi.controller.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + - name: node-driver-registrar + image: {{ .Values.csi.sidecars.registrar.repository }}:{{ .Values.csi.sidecars.registrar.tag }} + args: + - --v=2 + - --csi-address=/csi/csi.sock + - --kubelet-registration-path=/var/lib/kubelet/plugins/{{ .Values.csi.driver.name }}/csi.sock + env: + - name: KUBE_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + volumeMounts: + - name: plugin-dir + mountPath: /csi + - name: registration-dir + mountPath: /registration + - name: kubelet-dir + mountPath: /var/lib/kubelet + mountPropagation: HostToContainer + {{- with .Values.csi.node.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + - name: csi-liveness-probe + image: {{ .Values.csi.sidecars.livenessProbe.repository }}:{{ .Values.csi.sidecars.livenessProbe.tag }} + args: + - --csi-address=/csi/csi.sock + - --health-port=9808 + volumeMounts: + - name: plugin-dir + mountPath: /csi + - name: lb-controller + image: {{ include "kks-provider-plugin.image" . }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + command: + - /kks-provider + - lb-controller + - --metrics-bind-address + - {{ .Values.metrics.lbController.bindAddress | quote }} + env: + - name: KLOUD_LB_API_URL + value: {{ .Values.lb.serverURL | quote }} + - name: KLOUD_LB_ACCESS_TOKEN + valueFrom: + secretKeyRef: + name: {{ include "kks-provider-plugin.lbSecretName" . }} + key: {{ .Values.lb.existingSecretAccessTokenKey }} + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + {{- with .Values.lb.controller.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + - name: lb-speaker + image: {{ include "kks-provider-plugin.image" . }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + command: + - /kks-provider + - lb-speaker + - --metrics-bind-address + - {{ .Values.metrics.lbSpeaker.bindAddress | quote }} + securityContext: + capabilities: + add: + - NET_ADMIN + - NET_RAW + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: KLOUD_LB_INTERFACE + value: {{ .Values.lb.speaker.interface | quote }} + {{- with .Values.lb.speaker.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + volumes: + - name: plugin-dir + hostPath: + path: /var/lib/kubelet/plugins/{{ .Values.csi.driver.name }}/ + type: DirectoryOrCreate + - name: registration-dir + hostPath: + path: /var/lib/kubelet/plugins_registry/ + type: Directory + - name: kubelet-dir + hostPath: + path: /var/lib/kubelet + type: Directory + - name: device-dir + hostPath: + path: /dev + - name: config + configMap: + name: {{ include "kks-provider-plugin.fullname" . }}-driver diff --git a/charts/kloud-csi/templates/rbac-controller.yaml b/charts/kks-provider-plugin/templates/rbac-controller.yaml similarity index 70% rename from charts/kloud-csi/templates/rbac-controller.yaml rename to charts/kks-provider-plugin/templates/rbac-controller.yaml index daf5897..84a5bea 100644 --- a/charts/kloud-csi/templates/rbac-controller.yaml +++ b/charts/kks-provider-plugin/templates/rbac-controller.yaml @@ -1,19 +1,10 @@ {{- if .Values.rbac.create }} -apiVersion: v1 -kind: ServiceAccount -metadata: - name: {{ include "kloud-csi.controllerServiceAccountName" . }} - namespace: {{ .Release.Namespace }} - labels: - {{- include "kloud-csi.labels" . | nindent 4 }} - app.kubernetes.io/component: controller ---- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: - name: {{ include "kloud-csi.fullname" . }}-controller + name: {{ include "kks-provider-plugin.fullname" . }}-csi-controller labels: - {{- include "kloud-csi.labels" . | nindent 4 }} + {{- include "kks-provider-plugin.labels" . | nindent 4 }} rules: - apiGroups: [""] resources: ["persistentvolumes"] @@ -49,15 +40,15 @@ rules: apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: - name: {{ include "kloud-csi.fullname" . }}-controller + name: {{ include "kks-provider-plugin.fullname" . }}-csi-controller labels: - {{- include "kloud-csi.labels" . | nindent 4 }} + {{- include "kks-provider-plugin.labels" . | nindent 4 }} roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: {{ include "kloud-csi.fullname" . }}-controller + name: {{ include "kks-provider-plugin.fullname" . }}-csi-controller subjects: - kind: ServiceAccount - name: {{ include "kloud-csi.controllerServiceAccountName" . }} + name: {{ include "kks-provider-plugin.lbServiceAccountName" . }} namespace: {{ .Release.Namespace }} {{- end }} diff --git a/charts/kks-provider-plugin/templates/rbac-node.yaml b/charts/kks-provider-plugin/templates/rbac-node.yaml new file mode 100644 index 0000000..8edbe44 --- /dev/null +++ b/charts/kks-provider-plugin/templates/rbac-node.yaml @@ -0,0 +1,27 @@ +{{- if .Values.rbac.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "kks-provider-plugin.fullname" . }}-csi-node + labels: + {{- include "kks-provider-plugin.labels" . | nindent 4 }} +rules: + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "kks-provider-plugin.fullname" . }}-csi-node + labels: + {{- include "kks-provider-plugin.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ include "kks-provider-plugin.fullname" . }}-csi-node +subjects: + - kind: ServiceAccount + name: {{ include "kks-provider-plugin.lbServiceAccountName" . }} + namespace: {{ .Release.Namespace }} +{{- end }} diff --git a/charts/kks-provider-plugin/templates/secret.yaml b/charts/kks-provider-plugin/templates/secret.yaml new file mode 100644 index 0000000..d5c69b2 --- /dev/null +++ b/charts/kks-provider-plugin/templates/secret.yaml @@ -0,0 +1,28 @@ +{{- include "kks-provider-plugin.validateRequired" . }} +{{- if not .Values.lb.existingSecret }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "kks-provider-plugin.lbSecretName" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "kks-provider-plugin.labels" . | nindent 4 }} +type: Opaque +stringData: + {{ .Values.lb.existingSecretAccessTokenKey }}: {{ required "lb.accessToken is required when lb.existingSecret is not set" .Values.lb.accessToken | quote }} +{{- end }} +{{- if and (not .Values.lb.existingSecret) (not .Values.csi.existingSecret) }} +--- +{{- end }} +{{- if not .Values.csi.existingSecret }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "kks-provider-plugin.csiSecretName" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "kks-provider-plugin.labels" . | nindent 4 }} +type: Opaque +stringData: + {{ .Values.csi.existingSecretAccessTokenKey }}: {{ required "csi.accessToken is required when csi.existingSecret is not set" .Values.csi.accessToken | quote }} +{{- end }} diff --git a/charts/kks-provider-plugin/templates/storageclass.yaml b/charts/kks-provider-plugin/templates/storageclass.yaml new file mode 100644 index 0000000..333a9e0 --- /dev/null +++ b/charts/kks-provider-plugin/templates/storageclass.yaml @@ -0,0 +1,49 @@ +{{- if .Values.csi.storageClass.enabled }} +{{- if .Values.csi.storageClass.immediate.enabled }} +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: {{ .Values.csi.storageClass.immediate.name }} + {{- if .Values.csi.storageClass.immediate.isDefault }} + annotations: + storageclass.kubernetes.io/is-default-class: "true" + {{- end }} + labels: + {{- include "kks-provider-plugin.labels" . | nindent 4 }} +provisioner: {{ .Values.csi.driver.name }} +reclaimPolicy: {{ .Values.csi.storageClass.reclaimPolicy }} +volumeBindingMode: Immediate +allowVolumeExpansion: {{ .Values.csi.storageClass.allowVolumeExpansion }} +{{- with .Values.csi.storageClass.immediate.parameters }} +parameters: + {{- range $key, $value := . }} + {{ $key | quote }}: {{ $value | quote }} + {{- end }} +{{- end }} +{{- end }} +{{- if and .Values.csi.storageClass.immediate.enabled .Values.csi.storageClass.waitForFirstConsumer.enabled }} +--- +{{- end }} +{{- if .Values.csi.storageClass.waitForFirstConsumer.enabled }} +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: {{ .Values.csi.storageClass.waitForFirstConsumer.name }} + {{- if .Values.csi.storageClass.waitForFirstConsumer.isDefault }} + annotations: + storageclass.kubernetes.io/is-default-class: "true" + {{- end }} + labels: + {{- include "kks-provider-plugin.labels" . | nindent 4 }} +provisioner: {{ .Values.csi.driver.name }} +reclaimPolicy: {{ .Values.csi.storageClass.reclaimPolicy }} +volumeBindingMode: WaitForFirstConsumer +allowVolumeExpansion: {{ .Values.csi.storageClass.allowVolumeExpansion }} +{{- with .Values.csi.storageClass.waitForFirstConsumer.parameters }} +parameters: + {{- range $key, $value := . }} + {{ $key | quote }}: {{ $value | quote }} + {{- end }} +{{- end }} +{{- end }} +{{- end }} diff --git a/charts/kks-provider-plugin/values.yaml b/charts/kks-provider-plugin/values.yaml new file mode 100644 index 0000000..e62ed2f --- /dev/null +++ b/charts/kks-provider-plugin/values.yaml @@ -0,0 +1,100 @@ +# kks-provider-plugin installs both: +# - KloudLB controller + speaker +# - Kloud CSI controller + node plugin + +nameOverride: "" +fullnameOverride: "" + +image: + repository: ghcr.io/kubelancloud/kks-provider-plugin + tag: "latest" + pullPolicy: IfNotPresent + +imagePullSecrets: [] + +lb: + serverURL: "https://lb.kloud.team" + accessToken: "" + existingSecret: "" + existingSecretAccessTokenKey: access-token + controller: + resources: + requests: + cpu: 25m + memory: 64Mi + limits: + cpu: 200m + memory: 128Mi + speaker: + interface: eth0 + resources: + requests: + cpu: 25m + memory: 64Mi + limits: + cpu: 200m + memory: 128Mi + serviceAccount: + create: true + name: "" + +csi: + serverURL: "https://storage.csi.kloud.team" + accessToken: "" + existingSecret: "" + existingSecretAccessTokenKey: access-token + driver: + name: storage.csi.kloud.team + sidecars: + provisioner: + repository: registry.k8s.io/sig-storage/csi-provisioner + tag: v5.1.0 + attacher: + repository: registry.k8s.io/sig-storage/csi-attacher + tag: v4.7.0 + registrar: + repository: registry.k8s.io/sig-storage/csi-node-driver-registrar + tag: v2.12.0 + livenessProbe: + repository: registry.k8s.io/sig-storage/livenessprobe + tag: v2.13.1 + storageClass: + enabled: true + reclaimPolicy: Delete + allowVolumeExpansion: false + immediate: + enabled: true + name: kloud-csi + isDefault: true + parameters: + kks.kloud/provisioning-mode: immediate + waitForFirstConsumer: + enabled: true + name: kloud-csi-wait-for-first-consumer + isDefault: false + parameters: + kks.kloud/provisioning-mode: deferred + controller: + resources: {} + node: + resources: {} + priorityClassName: system-node-critical + tolerations: + - operator: Exists + +rbac: + create: true + +podLabels: {} +podAnnotations: {} + +nodeSelector: {} +affinity: {} + +metrics: + csi: + bindAddress: ":10080" + lbController: + bindAddress: ":10081" + lbSpeaker: + bindAddress: ":10082" diff --git a/charts/kloud-csi/Chart.yaml b/charts/kloud-csi/Chart.yaml deleted file mode 100644 index ec2e0a7..0000000 --- a/charts/kloud-csi/Chart.yaml +++ /dev/null @@ -1,16 +0,0 @@ -apiVersion: v2 -name: kloud-csi -description: Kloud CSI driver for kks persistent volumes -type: application -version: 0.1.2 -appVersion: "0.1.0" -kubeVersion: ">=1.28.0-0" -home: https://github.com/KubelanCloud/kks-csi-plugin -sources: - - https://github.com/KubelanCloud/kks-csi-plugin -keywords: - - csi - - storage - - kloud -maintainers: - - name: Kloud Team diff --git a/charts/kloud-csi/templates/_helpers.tpl b/charts/kloud-csi/templates/_helpers.tpl deleted file mode 100644 index 134e244..0000000 --- a/charts/kloud-csi/templates/_helpers.tpl +++ /dev/null @@ -1,77 +0,0 @@ -{{/* -Expand the name of the chart. -*/}} -{{- define "kloud-csi.name" -}} -{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} -{{- end }} - -{{/* -Create a default fully qualified app name. -*/}} -{{- define "kloud-csi.fullname" -}} -{{- if .Values.fullnameOverride }} -{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- $name := default .Chart.Name .Values.nameOverride }} -{{- if contains $name .Release.Name }} -{{- .Release.Name | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} -{{- end }} -{{- end }} -{{- end }} - -{{- define "kloud-csi.chart" -}} -{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} -{{- end }} - -{{- define "kloud-csi.labels" -}} -helm.sh/chart: {{ include "kloud-csi.chart" . }} -{{ include "kloud-csi.selectorLabels" . }} -{{- if .Chart.AppVersion }} -app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} -{{- end }} -app.kubernetes.io/managed-by: {{ .Release.Service }} -{{- end }} - -{{- define "kloud-csi.selectorLabels" -}} -app.kubernetes.io/name: {{ include "kloud-csi.name" . }} -app.kubernetes.io/instance: {{ .Release.Name }} -{{- end }} - -{{- define "kloud-csi.controllerName" -}} -{{- printf "%s-controller" (include "kloud-csi.fullname" .) }} -{{- end }} - -{{- define "kloud-csi.nodeName" -}} -{{- printf "%s-node" (include "kloud-csi.fullname" .) }} -{{- end }} - -{{- define "kloud-csi.controllerServiceAccountName" -}} -{{- if .Values.serviceAccount.controller.create }} -{{- default (printf "%s-controller-sa" (include "kloud-csi.fullname" .)) .Values.serviceAccount.controller.name }} -{{- else }} -{{- default "default" .Values.serviceAccount.controller.name }} -{{- end }} -{{- end }} - -{{- define "kloud-csi.nodeServiceAccountName" -}} -{{- if .Values.serviceAccount.node.create }} -{{- default (printf "%s-node-sa" (include "kloud-csi.fullname" .)) .Values.serviceAccount.node.name }} -{{- else }} -{{- default "default" .Values.serviceAccount.node.name }} -{{- end }} -{{- end }} - -{{- define "kloud-csi.driverImage" -}} -{{- printf "%s:%s" .Values.image.repository (default .Chart.AppVersion .Values.image.tag) }} -{{- end }} - -{{- define "kloud-csi.validateRequired" -}} -{{- if and (not .Values.existingSecret) (not .Values.accessToken) }} -{{- fail "accessToken or existingSecret is required" }} -{{- end }} -{{- if not .Values.serverURL }} -{{- fail "serverURL is required" }} -{{- end }} -{{- end }} diff --git a/charts/kloud-csi/templates/controller-deployment.yaml b/charts/kloud-csi/templates/controller-deployment.yaml deleted file mode 100644 index c64d222..0000000 --- a/charts/kloud-csi/templates/controller-deployment.yaml +++ /dev/null @@ -1,108 +0,0 @@ -{{- include "kloud-csi.validateRequired" . }} -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ include "kloud-csi.controllerName" . }} - namespace: {{ .Release.Namespace }} - labels: - {{- include "kloud-csi.labels" . | nindent 4 }} - app.kubernetes.io/component: controller -spec: - replicas: {{ .Values.controller.replicas }} - selector: - matchLabels: - {{- include "kloud-csi.selectorLabels" . | nindent 6 }} - app.kubernetes.io/component: controller - template: - metadata: - labels: - {{- include "kloud-csi.selectorLabels" . | nindent 8 }} - app.kubernetes.io/component: controller - {{- with .Values.podLabels }} - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.podAnnotations }} - annotations: - {{- toYaml . | nindent 8 }} - {{- end }} - spec: - serviceAccountName: {{ include "kloud-csi.controllerServiceAccountName" . }} - {{- with .Values.imagePullSecrets }} - imagePullSecrets: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.nodeSelector }} - nodeSelector: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.affinity }} - affinity: - {{- toYaml . | nindent 8 }} - {{- end }} - containers: - - name: kloud-csi-plugin - image: {{ include "kloud-csi.driverImage" . }} - imagePullPolicy: {{ .Values.image.pullPolicy }} - command: - - /kks-csi - args: - - --config-file - - /etc/kloud-csi/driver.hcl - env: - - name: KKS_CSI_SERVER_URL - value: {{ .Values.serverURL | quote }} - - name: KKS_CSI_ACCESS_TOKEN - valueFrom: - secretKeyRef: - name: {{ default (printf "%s-config" (include "kloud-csi.fullname" .)) .Values.existingSecret }} - key: {{ .Values.existingSecretAccessTokenKey }} - - name: KKS_CSI_DRIVER_MODE - value: controller - - name: KKS_CSI_DRIVER_ENDPOINT - value: unix:///csi/csi.sock - - name: KKS_CSI_DRIVER_NAME - value: {{ .Values.driver.name | quote }} - volumeMounts: - - name: socket-dir - mountPath: /csi - - name: config - mountPath: /etc/kloud-csi - {{- with .Values.controller.resources }} - resources: - {{- toYaml . | nindent 12 }} - {{- end }} - - name: csi-provisioner - image: {{ .Values.sidecars.provisioner.repository }}:{{ .Values.sidecars.provisioner.tag }} - args: - - --csi-address=/csi/csi.sock - - --v=2 - - --feature-gates=Topology=false - - --timeout=60s - - --leader-election - - --default-fstype=ext4 - volumeMounts: - - name: socket-dir - mountPath: /csi - - name: csi-attacher - image: {{ .Values.sidecars.attacher.repository }}:{{ .Values.sidecars.attacher.tag }} - args: - - --csi-address=/csi/csi.sock - - --v=2 - - --leader-election - volumeMounts: - - name: socket-dir - mountPath: /csi - - name: liveness-probe - image: {{ .Values.sidecars.livenessProbe.repository }}:{{ .Values.sidecars.livenessProbe.tag }} - args: - - --csi-address=/csi/csi.sock - - --health-port=9808 - volumeMounts: - - name: socket-dir - mountPath: /csi - volumes: - - name: socket-dir - emptyDir: {} - - name: config - configMap: - name: {{ include "kloud-csi.fullname" . }}-driver diff --git a/charts/kloud-csi/templates/node-daemonset.yaml b/charts/kloud-csi/templates/node-daemonset.yaml deleted file mode 100644 index 105df46..0000000 --- a/charts/kloud-csi/templates/node-daemonset.yaml +++ /dev/null @@ -1,136 +0,0 @@ -{{- include "kloud-csi.validateRequired" . }} -apiVersion: apps/v1 -kind: DaemonSet -metadata: - name: {{ include "kloud-csi.nodeName" . }} - namespace: {{ .Release.Namespace }} - labels: - {{- include "kloud-csi.labels" . | nindent 4 }} - app.kubernetes.io/component: node -spec: - selector: - matchLabels: - {{- include "kloud-csi.selectorLabels" . | nindent 6 }} - app.kubernetes.io/component: node - template: - metadata: - labels: - {{- include "kloud-csi.selectorLabels" . | nindent 8 }} - app.kubernetes.io/component: node - {{- with .Values.podLabels }} - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.podAnnotations }} - annotations: - {{- toYaml . | nindent 8 }} - {{- end }} - spec: - serviceAccountName: {{ include "kloud-csi.nodeServiceAccountName" . }} - {{- with .Values.imagePullSecrets }} - imagePullSecrets: - {{- toYaml . | nindent 8 }} - {{- end }} - priorityClassName: {{ .Values.node.priorityClassName }} - hostNetwork: true - hostPID: true - {{- with .Values.node.tolerations }} - tolerations: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.nodeSelector }} - nodeSelector: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.affinity }} - affinity: - {{- toYaml . | nindent 8 }} - {{- end }} - containers: - - name: kloud-csi-plugin - image: {{ include "kloud-csi.driverImage" . }} - imagePullPolicy: {{ .Values.image.pullPolicy }} - securityContext: - privileged: true - command: - - /kks-csi - args: - - --config-file - - /etc/kloud-csi/driver.hcl - env: - - name: KKS_CSI_SERVER_URL - value: {{ .Values.serverURL | quote }} - - name: KKS_CSI_ACCESS_TOKEN - valueFrom: - secretKeyRef: - name: {{ default (printf "%s-config" (include "kloud-csi.fullname" .)) .Values.existingSecret }} - key: {{ .Values.existingSecretAccessTokenKey }} - - name: KKS_CSI_DRIVER_MODE - value: node - - name: KKS_CSI_DRIVER_ENDPOINT - value: unix:///csi/csi.sock - - name: KKS_CSI_DRIVER_NAME - value: {{ .Values.driver.name | quote }} - - name: KKS_CSI_NODE_ID - valueFrom: - fieldRef: - fieldPath: spec.nodeName - volumeMounts: - - name: plugin-dir - mountPath: /csi - - name: kubelet-dir - mountPath: /var/lib/kubelet - mountPropagation: Bidirectional - - name: device-dir - mountPath: /dev - - name: config - mountPath: /etc/kloud-csi - {{- with .Values.node.resources }} - resources: - {{- toYaml . | nindent 12 }} - {{- end }} - - name: node-driver-registrar - image: {{ .Values.sidecars.registrar.repository }}:{{ .Values.sidecars.registrar.tag }} - args: - - --v=2 - - --csi-address=/csi/csi.sock - - --kubelet-registration-path=/var/lib/kubelet/plugins/{{ .Values.driver.name }}/csi.sock - env: - - name: KUBE_NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - volumeMounts: - - name: plugin-dir - mountPath: /csi - - name: registration-dir - mountPath: /registration - - name: kubelet-dir - mountPath: /var/lib/kubelet - mountPropagation: HostToContainer - - name: liveness-probe - image: {{ .Values.sidecars.livenessProbe.repository }}:{{ .Values.sidecars.livenessProbe.tag }} - args: - - --csi-address=/csi/csi.sock - - --health-port=9809 - volumeMounts: - - name: plugin-dir - mountPath: /csi - volumes: - - name: plugin-dir - hostPath: - path: /var/lib/kubelet/plugins/{{ .Values.driver.name }}/ - type: DirectoryOrCreate - - name: registration-dir - hostPath: - path: /var/lib/kubelet/plugins_registry/ - type: Directory - - name: kubelet-dir - hostPath: - path: /var/lib/kubelet - type: Directory - - name: device-dir - hostPath: - path: /dev - - name: config - configMap: - name: {{ include "kloud-csi.fullname" . }}-driver diff --git a/charts/kloud-csi/templates/rbac-node.yaml b/charts/kloud-csi/templates/rbac-node.yaml deleted file mode 100644 index f31fa15..0000000 --- a/charts/kloud-csi/templates/rbac-node.yaml +++ /dev/null @@ -1,36 +0,0 @@ -{{- if .Values.rbac.create }} -apiVersion: v1 -kind: ServiceAccount -metadata: - name: {{ include "kloud-csi.nodeServiceAccountName" . }} - namespace: {{ .Release.Namespace }} - labels: - {{- include "kloud-csi.labels" . | nindent 4 }} - app.kubernetes.io/component: node ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: {{ include "kloud-csi.fullname" . }}-node - labels: - {{- include "kloud-csi.labels" . | nindent 4 }} -rules: - - apiGroups: [""] - resources: ["nodes"] - verbs: ["get", "list", "watch"] ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: {{ include "kloud-csi.fullname" . }}-node - labels: - {{- include "kloud-csi.labels" . | nindent 4 }} -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: {{ include "kloud-csi.fullname" . }}-node -subjects: - - kind: ServiceAccount - name: {{ include "kloud-csi.nodeServiceAccountName" . }} - namespace: {{ .Release.Namespace }} -{{- end }} diff --git a/charts/kloud-csi/templates/secret.yaml b/charts/kloud-csi/templates/secret.yaml deleted file mode 100644 index b0b59ca..0000000 --- a/charts/kloud-csi/templates/secret.yaml +++ /dev/null @@ -1,13 +0,0 @@ -{{- include "kloud-csi.validateRequired" . }} -{{- if not .Values.existingSecret }} -apiVersion: v1 -kind: Secret -metadata: - name: {{ include "kloud-csi.fullname" . }}-config - namespace: {{ .Release.Namespace }} - labels: - {{- include "kloud-csi.labels" . | nindent 4 }} -type: Opaque -stringData: - access-token: {{ required "accessToken is required when existingSecret is not set" .Values.accessToken | quote }} -{{- end }} diff --git a/charts/kloud-csi/templates/storageclass.yaml b/charts/kloud-csi/templates/storageclass.yaml deleted file mode 100644 index 1c3c56c..0000000 --- a/charts/kloud-csi/templates/storageclass.yaml +++ /dev/null @@ -1,49 +0,0 @@ -{{- if .Values.storageClass.enabled }} -{{- if .Values.storageClass.immediate.enabled }} -apiVersion: storage.k8s.io/v1 -kind: StorageClass -metadata: - name: {{ .Values.storageClass.immediate.name }} - {{- if .Values.storageClass.immediate.isDefault }} - annotations: - storageclass.kubernetes.io/is-default-class: "true" - {{- end }} - labels: - {{- include "kloud-csi.labels" . | nindent 4 }} -provisioner: {{ .Values.driver.name }} -reclaimPolicy: {{ .Values.storageClass.reclaimPolicy }} -volumeBindingMode: Immediate -allowVolumeExpansion: {{ .Values.storageClass.allowVolumeExpansion }} -{{- with .Values.storageClass.immediate.parameters }} -parameters: - {{- range $key, $value := . }} - {{ $key | quote }}: {{ $value | quote }} - {{- end }} -{{- end }} -{{- end }} -{{- if and .Values.storageClass.immediate.enabled .Values.storageClass.waitForFirstConsumer.enabled }} ---- -{{- end }} -{{- if .Values.storageClass.waitForFirstConsumer.enabled }} -apiVersion: storage.k8s.io/v1 -kind: StorageClass -metadata: - name: {{ .Values.storageClass.waitForFirstConsumer.name }} - {{- if .Values.storageClass.waitForFirstConsumer.isDefault }} - annotations: - storageclass.kubernetes.io/is-default-class: "true" - {{- end }} - labels: - {{- include "kloud-csi.labels" . | nindent 4 }} -provisioner: {{ .Values.driver.name }} -reclaimPolicy: {{ .Values.storageClass.reclaimPolicy }} -volumeBindingMode: WaitForFirstConsumer -allowVolumeExpansion: {{ .Values.storageClass.allowVolumeExpansion }} -{{- with .Values.storageClass.waitForFirstConsumer.parameters }} -parameters: - {{- range $key, $value := . }} - {{ $key | quote }}: {{ $value | quote }} - {{- end }} -{{- end }} -{{- end }} -{{- end }} diff --git a/charts/kloud-csi/values.yaml b/charts/kloud-csi/values.yaml deleted file mode 100644 index 4fd737f..0000000 --- a/charts/kloud-csi/values.yaml +++ /dev/null @@ -1,87 +0,0 @@ -# Kloud CSI driver — install on each user Kubernetes cluster. -# -# Required: -# serverURL — URL of the management CSI server (from your Kloud cluster details) -# accessToken — cluster csi_access_token (create a Kubernetes secret or set here) -# -# Example: -# helm install kloud-csi ./charts/kloud-csi \ -# --namespace kube-system \ -# --set serverURL=http://10.0.0.5:9766 \ -# --set accessToken="$(kubectl get secret ... -o jsonpath='{.data.token}' | base64 -d)" - -nameOverride: "" -fullnameOverride: "" - -image: - repository: ghcr.io/kubelancloud/kks-csi-plugin - tag: "latest" - pullPolicy: Always - -imagePullSecrets: [] - -serverURL: "https://storage.csi.kloud.team" -accessToken: "" -existingSecret: "" -existingSecretAccessTokenKey: access-token - -driver: - name: storage.csi.kloud.team - -sidecars: - provisioner: - repository: registry.k8s.io/sig-storage/csi-provisioner - tag: v5.1.0 - attacher: - repository: registry.k8s.io/sig-storage/csi-attacher - tag: v4.7.0 - registrar: - repository: registry.k8s.io/sig-storage/csi-node-driver-registrar - tag: v2.12.0 - livenessProbe: - repository: registry.k8s.io/sig-storage/livenessprobe - tag: v2.13.1 - -storageClass: - enabled: true - reclaimPolicy: Delete - allowVolumeExpansion: false - immediate: - enabled: true - name: kloud-csi - isDefault: true - parameters: - kks.kloud/provisioning-mode: immediate - waitForFirstConsumer: - enabled: true - name: kloud-csi-wait-for-first-consumer - isDefault: false - parameters: - kks.kloud/provisioning-mode: deferred - -controller: - replicas: 1 - resources: {} - -node: - resources: {} - priorityClassName: system-node-critical - tolerations: - - operator: Exists - -serviceAccount: - controller: - create: true - name: "" - node: - create: true - name: "" - -rbac: - create: true - -podLabels: {} -podAnnotations: {} - -nodeSelector: {} -affinity: {} diff --git a/examples/csi-client.hcl b/examples/csi-client.hcl index 58527bc..2795720 100644 --- a/examples/csi-client.hcl +++ b/examples/csi-client.hcl @@ -1,7 +1,7 @@ # Run on user cluster nodes via Helm, or standalone with: -# kks-csi -c examples/csi-client.hcl +# kks-provider csi -c examples/csi-client.hcl # -# When installed with charts/kloud-csi, settings come from env vars instead. +# When installed with charts/kks-provider-plugin, settings come from env vars instead. driver { name = "storage.csi.kloud.team" endpoint = "unix:///var/lib/kubelet/plugins/storage.csi.kloud.team/csi.sock" diff --git a/go.mod b/go.mod index db7cb4f..ed26796 100644 --- a/go.mod +++ b/go.mod @@ -1,14 +1,19 @@ -module github.com/KubelanCloud/kks-csi-plugin +module github.com/KubelanCloud/kks-provider-plugin go 1.24.5 require ( github.com/container-storage-interface/spec v1.9.0 github.com/hashicorp/hcl/v2 v2.24.0 + github.com/prometheus/client_golang v1.22.0 github.com/spf13/cobra v1.9.1 + github.com/vishvananda/netlink v1.3.1 go.uber.org/zap v1.27.0 google.golang.org/grpc v1.68.1 - google.golang.org/protobuf v1.34.2 + google.golang.org/protobuf v1.36.5 + k8s.io/api v0.33.3 + k8s.io/apimachinery v0.33.3 + k8s.io/client-go v0.33.3 k8s.io/mount-utils v0.33.3 k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 ) @@ -16,23 +21,57 @@ require ( require ( github.com/agext/levenshtein v1.2.1 // indirect github.com/apparentlymart/go-textseg/v15 v15.0.0 // indirect + github.com/beorn7/perks v1.0.1 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/emicklei/go-restful/v3 v3.11.0 // indirect + github.com/fxamacker/cbor/v2 v2.7.0 // indirect github.com/go-logr/logr v1.4.2 // indirect + github.com/go-openapi/jsonpointer v0.21.0 // indirect + github.com/go-openapi/jsonreference v0.20.2 // indirect + github.com/go-openapi/swag v0.23.0 // indirect + github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/protobuf v1.5.4 // indirect - github.com/google/go-cmp v0.6.0 // indirect + github.com/google/gnostic-models v0.6.9 // indirect + github.com/google/go-cmp v0.7.0 // indirect + github.com/google/uuid v1.6.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/josharian/intern v1.0.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/mailru/easyjson v0.7.7 // indirect github.com/mitchellh/go-wordwrap v1.0.1 // indirect github.com/moby/sys/mountinfo v0.7.2 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/prometheus/client_model v0.6.1 // indirect + github.com/prometheus/common v0.62.0 // indirect + github.com/prometheus/procfs v0.15.1 // indirect github.com/spf13/pflag v1.0.6 // indirect + github.com/vishvananda/netns v0.0.5 // indirect + github.com/x448/float16 v0.8.4 // indirect github.com/zclconf/go-cty v1.16.3 // indirect go.uber.org/multierr v1.10.0 // indirect - golang.org/x/mod v0.17.0 // indirect - golang.org/x/net v0.29.0 // indirect + golang.org/x/mod v0.21.0 // indirect + golang.org/x/net v0.38.0 // indirect + golang.org/x/oauth2 v0.27.0 // indirect golang.org/x/sync v0.14.0 // indirect golang.org/x/sys v0.33.0 // indirect + golang.org/x/term v0.32.0 // indirect golang.org/x/text v0.25.0 // indirect - golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect + golang.org/x/time v0.9.0 // indirect + golang.org/x/tools v0.26.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 // indirect + gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect + gopkg.in/inf.v0 v0.9.1 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/klog/v2 v2.130.1 // indirect + k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff // indirect + sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 // indirect + sigs.k8s.io/randfill v1.0.0 // indirect + sigs.k8s.io/structured-merge-diff/v4 v4.6.0 // indirect + sigs.k8s.io/yaml v1.4.0 // indirect ) replace k8s.io/kubernetes => github.com/kubernetes/kubernetes v1.33.3 diff --git a/go.sum b/go.sum index a9d4bcf..8867626 100644 --- a/go.sum +++ b/go.sum @@ -2,36 +2,125 @@ github.com/agext/levenshtein v1.2.1 h1:QmvMAjj2aEICytGiWzmxoE0x2KZvE0fvmqMOfy2tj github.com/agext/levenshtein v1.2.1/go.mod h1:JEDfjyjHDjOF/1e4FlBE/PkbqA9OfWu2ki2W0IB5558= github.com/apparentlymart/go-textseg/v15 v15.0.0 h1:uYvfpb3DyLSCGWnctWKGj857c6ew1u1fNQOlOtuGxQY= github.com/apparentlymart/go-textseg/v15 v15.0.0/go.mod h1:K8XmNZdhEBkdlyDdvbmmsvpAG721bKi0joRfFdHIWJ4= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/container-storage-interface/spec v1.9.0 h1:zKtX4STsq31Knz3gciCYCi1SXtO2HJDecIjDVboYavY= github.com/container-storage-interface/spec v1.9.0/go.mod h1:ZfDu+3ZRyeVqxZM0Ds19MVLkN2d1XJ5MAfi1L3VjlT0= github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g= +github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E= +github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ= github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= +github.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ= +github.com/go-openapi/jsonpointer v0.21.0/go.mod h1:IUyH9l/+uyhIYQ/PXVA41Rexl+kOkAPDdXEYns6fzUY= +github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE= +github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= +github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= +github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+GrE= +github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ= +github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= +github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= github.com/go-test/deep v1.0.3 h1:ZrJSEWsXzPOxaZnFteGEfooLba+ju3FYIbOrS+rQd68= github.com/go-test/deep v1.0.3/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= -github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= -github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/gnostic-models v0.6.9 h1:MU/8wDLif2qCXZmzncUQ/BOfxWfthHi63KqpoNbWqVw= +github.com/google/gnostic-models v0.6.9/go.mod h1:CiWsm0s6BSQd1hRn8/QmxqB6BesYcbSZxsz9b0KuDBw= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db h1:097atOisP2aRj7vFgYQBbFN4U4JNXUNYpxael3UzMyo= +github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/hashicorp/hcl/v2 v2.24.0 h1:2QJdZ454DSsYGoaE6QheQZjtKZSUs9Nh2izTWiwQxvE= github.com/hashicorp/hcl/v2 v2.24.0/go.mod h1:oGoO1FIQYfn/AgyOhlg9qLC6/nOJPX3qGbkZpYAcqfM= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= +github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/mitchellh/go-wordwrap v1.0.1 h1:TLuKupo69TCn6TQSyGxwI1EblZZEsQ0vMlAFQflz0v0= github.com/mitchellh/go-wordwrap v1.0.1/go.mod h1:R62XHJLzvMFRBbcrT7m7WgmE1eOyTSsCt+hzestvNj0= github.com/moby/sys/mountinfo v0.7.2 h1:1shs6aH5s4o5H2zQLn796ADW1wMrIwHsyJ2v9KouLrg= github.com/moby/sys/mountinfo v0.7.2/go.mod h1:1YOa8w8Ih7uW0wALDUgT1dTTSBrZ+HiBLGws92L2RU4= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/onsi/ginkgo/v2 v2.21.0 h1:7rg/4f3rB88pb5obDgNZrNHrQ4e6WpjonchcpuBRnZM= +github.com/onsi/ginkgo/v2 v2.21.0/go.mod h1:7Du3c42kxCUegi0IImZ1wUQzMBVecgIHjR1C+NkhLQo= +github.com/onsi/gomega v1.35.1 h1:Cwbd75ZBPxFSuZ6T+rN/WCb/gOc6YgFBXLlZLhC7Ds4= +github.com/onsi/gomega v1.35.1/go.mod h1:PvZbdDc8J6XJEpDK4HCuRBm8a6Fzp9/DmhC9C7yFlog= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v1.22.0 h1:rb93p9lokFEsctTys46VnV1kLCDpVZ0a/Y92Vm0Zc6Q= +github.com/prometheus/client_golang v1.22.0/go.mod h1:R7ljNsLXhuQXYZYtw6GAE9AZg8Y7vEW5scdCXrWRXC0= +github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= +github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= +github.com/prometheus/common v0.62.0 h1:xasJaQlnWAeyHdUBeGjXmutelfJHWMRr+Fg4QszZ2Io= +github.com/prometheus/common v0.62.0/go.mod h1:vyBcEuLSvWos9B1+CyL7JZ2up+uFzXhkqml0W5zIY1I= +github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= +github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= +github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= +github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/spf13/cobra v1.9.1 h1:CXSaggrXdbHK9CF+8ywj8Amf7PBRmPCOJugH954Nnlo= github.com/spf13/cobra v1.9.1/go.mod h1:nDyEzZ8ogv936Cinf6g1RU9MRY64Ir93oCnqb9wxYW0= github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o= github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/vishvananda/netlink v1.3.1 h1:3AEMt62VKqz90r0tmNhog0r/PpWKmrEShJU0wJW6bV0= +github.com/vishvananda/netlink v1.3.1/go.mod h1:ARtKouGSTGchR8aMwmkzC0qiNPrrWO5JS/XMVl45+b4= +github.com/vishvananda/netns v0.0.5 h1:DfiHV+j8bA32MFM7bfEunvT8IAqQ/NzSJHtcmW5zdEY= +github.com/vishvananda/netns v0.0.5/go.mod h1:SpkAiCQRtJ6TvvxPnOSyH3BMl6unz3xZlaprSwhNNJM= +github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= +github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/zclconf/go-cty v1.16.3 h1:osr++gw2T61A8KVYHoQiFbFd1Lh3JOCXc/jFLJXKTxk= github.com/zclconf/go-cty v1.16.3/go.mod h1:VvMs5i0vgZdhYawQNq5kePSpLAoz8u1xvZgrPIxfnZE= github.com/zclconf/go-cty-debug v0.0.0-20240509010212-0d6042c53940 h1:4r45xpDWB6ZMSMNJFMOjqrGHynW3DIBuR2H9j0ug+Mo= @@ -42,32 +131,89 @@ go.uber.org/multierr v1.10.0 h1:S0h4aNzvfcFsC3dRF1jLoaov7oRaKqRGC/pUEJ2yvPQ= go.uber.org/multierr v1.10.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8= golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY= -golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA= -golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= -golang.org/x/net v0.29.0 h1:5ORfpBpCs4HzDYoodCDBbwHzdR5UrLBZ3sOnUJmFoHo= -golang.org/x/net v0.29.0/go.mod h1:gLkgy8jTGERgjzMic6DS9+SP0ajcu6Xu3Orq/SpETg0= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.21.0 h1:vvrHzRwRfVKSiLrG+d4FMl/Qi4ukBCE6kZlTUkDYRT0= +golang.org/x/mod v0.21.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8= +golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= +golang.org/x/oauth2 v0.27.0 h1:da9Vo7/tDv5RH/7nZDz1eMGS/q1Vv1N/7FCrBhI9I3M= +golang.org/x/oauth2 v0.27.0/go.mod h1:onh5ek6nERTohokkhCD/y2cV4Do3fxFHFuAejCkRWT8= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.14.0 h1:woo0S4Yywslg6hp4eUFjTVOyKt0RookbpAHG4c1HmhQ= golang.org/x/sync v0.14.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw= golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/term v0.32.0 h1:DR4lr0TjUs3epypdhTOkMmuF5CDFJ/8pOnbzMZPQ7bg= +golang.org/x/term v0.32.0/go.mod h1:uZG1FhGx848Sqfsq4/DlJr3xGGsYMu/L5GW4abiaEPQ= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.25.0 h1:qVyWApTSYLk/drJRO5mDlNYskwQznZmkpV2c8q9zls4= golang.org/x/text v0.25.0/go.mod h1:WEdwpYrmk1qmdHvhkSTNPm3app7v4rsT8F2UD6+VHIA= -golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg= -golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= +golang.org/x/time v0.9.0 h1:EsRrnYcQiGH+5FfbgvV4AP7qEZstoyrHB0DzarOQ4ZY= +golang.org/x/time v0.9.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.26.0 h1:v/60pFQmzmT9ExmjDv2gGIfi3OqfKoEP6I5+umXlbnQ= +golang.org/x/tools v0.26.0/go.mod h1:TPVVj70c7JJ3WCazhD8OdXcZg/og+b9+tH/KxylGwH0= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 h1:pPJltXNxVzT4pK9yD8vR9X75DaWYYmLGMsEvBfFQZzQ= google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1/go.mod h1:UqMtugtsSgubUsoxbuAoiCXvqvErP7Gf0so0mK9tHxU= google.golang.org/grpc v1.68.1 h1:oI5oTa11+ng8r8XMMN7jAOmWfPZWbYpCFaMUTACxkM0= google.golang.org/grpc v1.68.1/go.mod h1:+q1XYFJjShcqn0QZHvCyeR4CXPA+llXIeUIfIe00waw= -google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= -google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= +google.golang.org/protobuf v1.36.5 h1:tPhr+woSbjfYvY6/GPufUoYizxw1cF/yFoxJ2fmpwlM= +google.golang.org/protobuf v1.36.5/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/evanphx/json-patch.v4 v4.12.0 h1:n6jtcsulIzXPJaxegRbvFNNrZDjbij7ny3gmSPG+6V4= +gopkg.in/evanphx/json-patch.v4 v4.12.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= +gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +k8s.io/api v0.33.3 h1:SRd5t//hhkI1buzxb288fy2xvjubstenEKL9K51KBI8= +k8s.io/api v0.33.3/go.mod h1:01Y/iLUjNBM3TAvypct7DIj0M0NIZc+PzAHCIo0CYGE= +k8s.io/apimachinery v0.33.3 h1:4ZSrmNa0c/ZpZJhAgRdcsFcZOw1PQU1bALVQ0B3I5LA= +k8s.io/apimachinery v0.33.3/go.mod h1:BHW0YOu7n22fFv/JkYOEfkUYNRN0fj0BlvMFWA7b+SM= +k8s.io/client-go v0.33.3 h1:M5AfDnKfYmVJif92ngN532gFqakcGi6RvaOF16efrpA= +k8s.io/client-go v0.33.3/go.mod h1:luqKBQggEf3shbxHY4uVENAxrDISLOarxpTKMiUuujg= k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= +k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff h1:/usPimJzUKKu+m+TE36gUyGcf03XZEP0ZIKgKj35LS4= +k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff/go.mod h1:5jIi+8yX4RIb8wk3XwBo5Pq2ccx4FP10ohkbSKCZoK8= k8s.io/mount-utils v0.33.3 h1:Q1jsnqdS4LdtJSYSXgiQv/XNrRHQncLk3gMYjKNSZrE= k8s.io/mount-utils v0.33.3/go.mod h1:1JR4rKymg8B8bCPo618hpSAdrpO6XLh0Acqok/xVwPE= k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 h1:M3sRQVHv7vB20Xc2ybTt7ODCeFj6JSWYFzOFnYeS6Ro= k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 h1:/Rv+M11QRah1itp8VhT6HoVx1Ray9eB4DBr+K+/sCJ8= +sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3/go.mod h1:18nIHnGi6636UCz6m8i4DhaJ65T6EruyzmoQqI2BVDo= +sigs.k8s.io/randfill v0.0.0-20250304075658-069ef1bbf016/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= +sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU= +sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= +sigs.k8s.io/structured-merge-diff/v4 v4.6.0 h1:IUA9nvMmnKWcj5jl84xn+T5MnlZKThmUW1TdblaLVAc= +sigs.k8s.io/structured-merge-diff/v4 v4.6.0/go.mod h1:dDy58f92j70zLsuZVuUX5Wp9vtxXpaZnkPGWeqDfCps= +sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= +sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY= diff --git a/main.go b/main.go index 9f07e57..5e65f7a 100644 --- a/main.go +++ b/main.go @@ -2,15 +2,27 @@ package main import ( "context" + "errors" "fmt" + "net" + "net/http" "os" "os/signal" + "strings" "syscall" + "time" - "github.com/KubelanCloud/kks-csi-plugin/config" - "github.com/KubelanCloud/kks-csi-plugin/pkg/csi/driver" + "github.com/KubelanCloud/kks-provider-plugin/config" + "github.com/KubelanCloud/kks-provider-plugin/pkg/csi/driver" + "github.com/KubelanCloud/kks-provider-plugin/pkg/kloudlb/controller" + "github.com/KubelanCloud/kks-provider-plugin/pkg/kloudlb/speaker" + lbapi "github.com/KubelanCloud/kks-provider-plugin/pkg/lb/api" + "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/spf13/cobra" "go.uber.org/zap" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" ) func main() { @@ -21,24 +33,116 @@ func main() { } defer logger.Sync() //nolint:errcheck - configPath := "csi.hcl" - rootCmd := &cobra.Command{ - Use: "kks-csi", - Short: "Kloud CSI driver for user cluster nodes", - Long: "Runs the in-cluster CSI gRPC driver. Storage operations are sent to the kks management CSI server.", - RunE: func(cmd *cobra.Command, args []string) error { - return run(cmd, configPath, logger) - }, + Use: "kks-provider", + Short: "Kloud provider plugin for user cluster nodes", + Long: "Runs the in-cluster provider components for CSI and LoadBalancer integration with kks management services.", } - rootCmd.PersistentFlags().StringVarP(&configPath, "config-file", "c", "csi.hcl", "Path to driver config (optional when using env vars)") + + rootCmd.AddCommand(csiCmd(logger), lbControllerCmd(logger), lbSpeakerCmd(logger)) if err := rootCmd.Execute(); err != nil { os.Exit(1) } } -func run(cmd *cobra.Command, configPath string, logger *zap.Logger) error { +func csiCmd(logger *zap.Logger) *cobra.Command { + configPath := "csi.hcl" + metricsBindAddress := ":10080" + cmd := &cobra.Command{ + Use: "csi", + Short: "Run the Kloud CSI driver", + RunE: func(cmd *cobra.Command, args []string) error { + return runCSI(cmd, configPath, metricsBindAddress, logger) + }, + } + cmd.Flags().StringVarP(&configPath, "config-file", "c", "csi.hcl", "Path to driver config (optional when using env vars)") + cmd.Flags().StringVar(&metricsBindAddress, "metrics-bind-address", ":10080", "Address to bind metrics server (set to 'off' to disable)") + return cmd +} + +func lbControllerCmd(logger *zap.Logger) *cobra.Command { + var ( + apiURL string + token string + metricsBindAddress string + ) + metricsBindAddress = ":10081" + cmd := &cobra.Command{ + Use: "lb-controller", + Short: "Run the KloudLB Service controller", + RunE: func(cmd *cobra.Command, args []string) error { + apiURL = envOr(apiURL, "KLOUD_LB_API_URL") + token = envOr(token, "KLOUD_LB_ACCESS_TOKEN") + if apiURL == "" { + return fmt.Errorf("KLOUD_LB_API_URL is required") + } + if token == "" { + return fmt.Errorf("KLOUD_LB_ACCESS_TOKEN is required") + } + + cfg, err := restConfig() + if err != nil { + return err + } + client, err := kubernetes.NewForConfig(cfg) + if err != nil { + return err + } + + lbClient := lbapi.NewClient(lbapi.ClientConfig{BaseURL: apiURL, Token: token, Timeout: 30 * time.Second}) + ctrl, err := controller.New(client, lbClient) + if err != nil { + return err + } + + ctx, cancel := signalContext(cmd.Context()) + defer cancel() + if err := startMetricsServer(ctx, logger, metricsBindAddress); err != nil { + return err + } + return ctrl.Run(ctx, 2) + }, + } + cmd.Flags().StringVar(&apiURL, "api-url", "", "KKS LoadBalancer API base URL") + cmd.Flags().StringVar(&token, "access-token", "", "Cluster LB access token") + cmd.Flags().StringVar(&metricsBindAddress, "metrics-bind-address", ":10081", "Address to bind metrics server (set to 'off' to disable)") + return cmd +} + +func lbSpeakerCmd(logger *zap.Logger) *cobra.Command { + metricsBindAddress := ":10082" + cmd := &cobra.Command{ + Use: "lb-speaker", + Short: "Run the KloudLB L2 speaker", + RunE: func(cmd *cobra.Command, args []string) error { + cfg, err := restConfig() + if err != nil { + return err + } + client, err := kubernetes.NewForConfig(cfg) + if err != nil { + return err + } + + spk, err := speaker.New(client, speaker.NodeNameFromEnv(), speaker.InterfaceFromEnv()) + if err != nil { + return err + } + + ctx, cancel := signalContext(cmd.Context()) + defer cancel() + if err := startMetricsServer(ctx, logger, metricsBindAddress); err != nil { + return err + } + return spk.Run(ctx) + }, + } + cmd.Flags().StringVar(&metricsBindAddress, "metrics-bind-address", ":10082", "Address to bind metrics server (set to 'off' to disable)") + return cmd +} + +func runCSI(cmd *cobra.Command, configPath, metricsBindAddress string, logger *zap.Logger) error { cfg, err := config.LoadClient(configPath) if err != nil { return fmt.Errorf("load config: %w", err) @@ -46,11 +150,75 @@ func run(cmd *cobra.Command, configPath string, logger *zap.Logger) error { ctx, cancel := signalContext(cmd.Context()) defer cancel() + if err := startMetricsServer(ctx, logger, metricsBindAddress); err != nil { + return err + } logger.Sugar().Infof("loaded csi driver config from %s", configPath) return driver.Run(ctx, cfg, logger) } +func startMetricsServer(ctx context.Context, logger *zap.Logger, bindAddress string) error { + bindAddress = strings.TrimSpace(bindAddress) + if bindAddress == "" || strings.EqualFold(bindAddress, "off") { + logger.Info("metrics server disabled") + return nil + } + + listener, err := net.Listen("tcp", bindAddress) + if err != nil { + return fmt.Errorf("listen metrics server on %s: %w", bindAddress, err) + } + + server := &http.Server{ + Addr: bindAddress, + Handler: metricsMux(), + ReadHeaderTimeout: 5 * time.Second, + } + + go func() { + <-ctx.Done() + shutdownCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if err := server.Shutdown(shutdownCtx); err != nil && !errors.Is(err, http.ErrServerClosed) { + logger.Error("failed to shutdown metrics server", zap.Error(err)) + } + }() + + go func() { + logger.Sugar().Infof("metrics server listening on %s", bindAddress) + if err := server.Serve(listener); err != nil && !errors.Is(err, http.ErrServerClosed) { + logger.Error("metrics server stopped unexpectedly", zap.Error(err)) + } + }() + + return nil +} + +func metricsMux() *http.ServeMux { + mux := http.NewServeMux() + mux.Handle("/metrics", promhttp.Handler()) + return mux +} + +func restConfig() (*rest.Config, error) { + if cfg, err := rest.InClusterConfig(); err == nil { + return cfg, nil + } + kubeconfig := os.Getenv("KUBECONFIG") + if kubeconfig == "" { + kubeconfig = clientcmd.RecommendedHomeFile + } + return clientcmd.BuildConfigFromFlags("", kubeconfig) +} + +func envOr(flagValue, envKey string) string { + if flagValue != "" { + return flagValue + } + return os.Getenv(envKey) +} + func signalContext(parent context.Context) (context.Context, context.CancelFunc) { ctx, cancel := context.WithCancel(parent) go func() { diff --git a/pkg/csi/api/client.go b/pkg/csi/api/client.go index 01fdfdd..1eae446 100644 --- a/pkg/csi/api/client.go +++ b/pkg/csi/api/client.go @@ -10,7 +10,7 @@ import ( "strings" "time" - "github.com/KubelanCloud/kks-csi-plugin/pkg/csi/provisioner" + "github.com/KubelanCloud/kks-provider-plugin/pkg/csi/provisioner" ) type ClientConfig struct { diff --git a/pkg/csi/api/client_test.go b/pkg/csi/api/client_test.go index 21eef93..110da7a 100644 --- a/pkg/csi/api/client_test.go +++ b/pkg/csi/api/client_test.go @@ -8,7 +8,7 @@ import ( "net/http/httptest" "testing" - "github.com/KubelanCloud/kks-csi-plugin/pkg/csi/provisioner" + "github.com/KubelanCloud/kks-provider-plugin/pkg/csi/provisioner" ) func TestClientClusterInfo(t *testing.T) { diff --git a/pkg/csi/client/backend.go b/pkg/csi/client/backend.go index 34a78e9..9bbe2b3 100644 --- a/pkg/csi/client/backend.go +++ b/pkg/csi/client/backend.go @@ -1,9 +1,9 @@ package client import ( - "github.com/KubelanCloud/kks-csi-plugin/config" - "github.com/KubelanCloud/kks-csi-plugin/pkg/csi/api" - "github.com/KubelanCloud/kks-csi-plugin/pkg/csi/provisioner" + "github.com/KubelanCloud/kks-provider-plugin/config" + "github.com/KubelanCloud/kks-provider-plugin/pkg/csi/api" + "github.com/KubelanCloud/kks-provider-plugin/pkg/csi/provisioner" ) func NewBackend(cfg *config.ClientConf) provisioner.Backend { diff --git a/pkg/csi/driver/controller.go b/pkg/csi/driver/controller.go index 69aac04..9c82dd8 100644 --- a/pkg/csi/driver/controller.go +++ b/pkg/csi/driver/controller.go @@ -3,7 +3,7 @@ package driver import ( "context" - "github.com/KubelanCloud/kks-csi-plugin/pkg/csi/provisioner" + "github.com/KubelanCloud/kks-provider-plugin/pkg/csi/provisioner" "github.com/container-storage-interface/spec/lib/go/csi" ) diff --git a/pkg/csi/driver/controller_test.go b/pkg/csi/driver/controller_test.go index 31f47a2..ac7ebbc 100644 --- a/pkg/csi/driver/controller_test.go +++ b/pkg/csi/driver/controller_test.go @@ -4,7 +4,7 @@ import ( "context" "testing" - "github.com/KubelanCloud/kks-csi-plugin/pkg/csi/provisioner" + "github.com/KubelanCloud/kks-provider-plugin/pkg/csi/provisioner" "github.com/container-storage-interface/spec/lib/go/csi" ) diff --git a/pkg/csi/driver/driver.go b/pkg/csi/driver/driver.go index 3f3225a..daae1d0 100644 --- a/pkg/csi/driver/driver.go +++ b/pkg/csi/driver/driver.go @@ -9,9 +9,9 @@ import ( "strings" "github.com/container-storage-interface/spec/lib/go/csi" - "github.com/KubelanCloud/kks-csi-plugin/config" - "github.com/KubelanCloud/kks-csi-plugin/pkg/csi/client" - "github.com/KubelanCloud/kks-csi-plugin/pkg/csi/provisioner" + "github.com/KubelanCloud/kks-provider-plugin/config" + "github.com/KubelanCloud/kks-provider-plugin/pkg/csi/client" + "github.com/KubelanCloud/kks-provider-plugin/pkg/csi/provisioner" "go.uber.org/zap" "google.golang.org/grpc" ) diff --git a/pkg/csi/driver/identity.go b/pkg/csi/driver/identity.go index 932d5e8..624ab53 100644 --- a/pkg/csi/driver/identity.go +++ b/pkg/csi/driver/identity.go @@ -4,7 +4,7 @@ import ( "context" "github.com/container-storage-interface/spec/lib/go/csi" - "github.com/KubelanCloud/kks-csi-plugin/config" + "github.com/KubelanCloud/kks-provider-plugin/config" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" "google.golang.org/protobuf/types/known/wrapperspb" diff --git a/pkg/csi/driver/node.go b/pkg/csi/driver/node.go index 888cba2..62d1a54 100644 --- a/pkg/csi/driver/node.go +++ b/pkg/csi/driver/node.go @@ -8,7 +8,7 @@ import ( "strings" "github.com/container-storage-interface/spec/lib/go/csi" - "github.com/KubelanCloud/kks-csi-plugin/pkg/csi/provisioner" + "github.com/KubelanCloud/kks-provider-plugin/pkg/csi/provisioner" ) type NodeServer struct { diff --git a/pkg/csi/driver/volume.go b/pkg/csi/driver/volume.go index e2707eb..6e4c83c 100644 --- a/pkg/csi/driver/volume.go +++ b/pkg/csi/driver/volume.go @@ -1,6 +1,6 @@ package driver -import "github.com/KubelanCloud/kks-csi-plugin/pkg/csi/volume" +import "github.com/KubelanCloud/kks-provider-plugin/pkg/csi/volume" func sanitizeVolumeName(name string) string { return volume.SanitizeName(name) diff --git a/pkg/kloudlb/constants.go b/pkg/kloudlb/constants.go new file mode 100644 index 0000000..da11fe4 --- /dev/null +++ b/pkg/kloudlb/constants.go @@ -0,0 +1,9 @@ +package kloudlb + +const ( + Finalizer = "lb.kloud.team/finalizer" + AnnotationIP = "lb.kloud.team/ip" + AnnotationLBID = "lb.kloud.team/id" + + Namespace = "kube-system" +) diff --git a/pkg/kloudlb/controller/controller.go b/pkg/kloudlb/controller/controller.go new file mode 100644 index 0000000..3526955 --- /dev/null +++ b/pkg/kloudlb/controller/controller.go @@ -0,0 +1,277 @@ +package controller + +import ( + "context" + "fmt" + "os" + "time" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/informers" + "k8s.io/client-go/kubernetes" + corelisters "k8s.io/client-go/listers/core/v1" + "k8s.io/client-go/tools/leaderelection" + "k8s.io/client-go/tools/leaderelection/resourcelock" + "k8s.io/client-go/tools/cache" + "k8s.io/client-go/util/workqueue" + + "github.com/KubelanCloud/kks-provider-plugin/pkg/kloudlb" + lbapi "github.com/KubelanCloud/kks-provider-plugin/pkg/lb/api" + "github.com/KubelanCloud/kks-provider-plugin/pkg/lb/provisioner" +) + +type Controller struct { + client kubernetes.Interface + lbClient *lbapi.Client + informer cache.SharedIndexInformer + lister corelisters.ServiceLister + queue workqueue.RateLimitingInterface + synced cache.InformerSynced + + identity string + leader *leaderelection.LeaderElector +} + +func New(client kubernetes.Interface, lbClient *lbapi.Client) (*Controller, error) { + if client == nil { + return nil, fmt.Errorf("kubernetes client is required") + } + if lbClient == nil { + return nil, fmt.Errorf("lb api client is required") + } + + factory := informers.NewSharedInformerFactory(client, 30*time.Second) + informer := factory.Core().V1().Services().Informer() + queue := workqueue.NewRateLimitingQueue(workqueue.DefaultControllerRateLimiter()) + + c := &Controller{ + client: client, + lbClient: lbClient, + informer: informer, + lister: factory.Core().V1().Services().Lister(), + queue: queue, + synced: informer.HasSynced, + identity: controllerIdentity(), + } + + informer.AddEventHandler(cache.ResourceEventHandlerFuncs{ + AddFunc: func(obj any) { c.enqueue(obj) }, + UpdateFunc: func(_, newObj any) { c.enqueue(newObj) }, + DeleteFunc: func(obj any) { c.enqueue(obj) }, + }) + + return c, nil +} + +func (c *Controller) Run(ctx context.Context, workers int) error { + if workers <= 0 { + workers = 1 + } + + go c.informer.Run(ctx.Done()) + if !cache.WaitForCacheSync(ctx.Done(), c.synced) { + return fmt.Errorf("service informer cache sync failed") + } + + elector, err := c.newLeaderElector() + if err != nil { + return err + } + c.leader = elector + go c.leader.Run(ctx) + + for i := 0; i < workers; i++ { + go wait.UntilWithContext(ctx, c.runWorker, time.Second) + } + + <-ctx.Done() + c.queue.ShutDown() + return nil +} + +func (c *Controller) enqueue(obj any) { + key, err := cache.MetaNamespaceKeyFunc(obj) + if err != nil { + return + } + c.queue.Add(key) +} + +func (c *Controller) runWorker(ctx context.Context) { + for c.processNext(ctx) { + } +} + +func (c *Controller) processNext(ctx context.Context) bool { + item, shutdown := c.queue.Get() + if shutdown { + return false + } + defer c.queue.Done(item) + key, ok := item.(string) + if !ok { + c.queue.Forget(item) + return true + } + + if err := c.sync(ctx, key); err != nil { + c.queue.AddRateLimited(item) + return true + } + c.queue.Forget(item) + return true +} + +func (c *Controller) sync(ctx context.Context, key string) error { + if !c.isLeader() { + return nil + } + + namespace, name, err := cache.SplitMetaNamespaceKey(key) + if err != nil { + return err + } + + svc, err := c.lister.Services(namespace).Get(name) + if apierrors.IsNotFound(err) { + return nil + } + if err != nil { + return err + } + + if svc.Spec.Type != corev1.ServiceTypeLoadBalancer { + return nil + } + + if svc.DeletionTimestamp != nil { + return c.finalize(ctx, svc) + } + + if !containsString(svc.Finalizers, kloudlb.Finalizer) { + patch := svc.DeepCopy() + patch.Finalizers = append(patch.Finalizers, kloudlb.Finalizer) + if _, err := c.client.CoreV1().Services(namespace).Update(ctx, patch, metav1.UpdateOptions{}); err != nil { + return err + } + return nil + } + + if ingressIP(svc) != "" { + return nil + } + + lb, err := c.lbClient.Allocate(ctx, provisioner.AllocateRequest{ + Namespace: namespace, + Name: name, + }) + if err != nil { + return err + } + + patch := svc.DeepCopy() + if patch.Annotations == nil { + patch.Annotations = map[string]string{} + } + patch.Annotations[kloudlb.AnnotationIP] = lb.IP + patch.Annotations[kloudlb.AnnotationLBID] = lb.ID + patch.Status = corev1.ServiceStatus{ + LoadBalancer: corev1.LoadBalancerStatus{ + Ingress: []corev1.LoadBalancerIngress{{IP: lb.IP}}, + }, + } + + _, err = c.client.CoreV1().Services(namespace).UpdateStatus(ctx, patch, metav1.UpdateOptions{}) + return err +} + +func (c *Controller) finalize(ctx context.Context, svc *corev1.Service) error { + if !containsString(svc.Finalizers, kloudlb.Finalizer) { + return nil + } + + if id := svc.Annotations[kloudlb.AnnotationLBID]; id != "" { + if err := c.lbClient.Release(ctx, id); err != nil { + return err + } + } + + patch := svc.DeepCopy() + patch.Finalizers = removeString(patch.Finalizers, kloudlb.Finalizer) + _, err := c.client.CoreV1().Services(svc.Namespace).Update(ctx, patch, metav1.UpdateOptions{}) + return err +} + +func ingressIP(svc *corev1.Service) string { + for _, ing := range svc.Status.LoadBalancer.Ingress { + if ing.IP != "" { + return ing.IP + } + } + if svc.Annotations != nil { + return svc.Annotations[kloudlb.AnnotationIP] + } + return "" +} + +func containsString(items []string, target string) bool { + for _, item := range items { + if item == target { + return true + } + } + return false +} + +func removeString(items []string, target string) []string { + out := make([]string, 0, len(items)) + for _, item := range items { + if item != target { + out = append(out, item) + } + } + return out +} + +func (c *Controller) isLeader() bool { + return c.leader != nil && c.leader.IsLeader() +} + +func (c *Controller) newLeaderElector() (*leaderelection.LeaderElector, error) { + lock := &resourcelock.LeaseLock{ + LeaseMeta: metav1.ObjectMeta{ + Namespace: kloudlb.Namespace, + Name: "kloud-lb-controller", + }, + Client: c.client.CoordinationV1(), + LockConfig: resourcelock.ResourceLockConfig{ + Identity: c.identity, + }, + } + + return leaderelection.NewLeaderElector(leaderelection.LeaderElectionConfig{ + Lock: lock, + LeaseDuration: 15 * time.Second, + RenewDeadline: 10 * time.Second, + RetryPeriod: 2 * time.Second, + ReleaseOnCancel: true, + Name: "kloud-lb-controller", + Callbacks: leaderelection.LeaderCallbacks{ + OnStartedLeading: func(context.Context) {}, + OnStoppedLeading: func() {}, + }, + }) +} + +func controllerIdentity() string { + if v := os.Getenv("NODE_NAME"); v != "" { + return v + } + if v, err := os.Hostname(); err == nil && v != "" { + return v + } + return fmt.Sprintf("controller-%d", time.Now().UnixNano()) +} diff --git a/pkg/kloudlb/controller/controller_test.go b/pkg/kloudlb/controller/controller_test.go new file mode 100644 index 0000000..c0cde14 --- /dev/null +++ b/pkg/kloudlb/controller/controller_test.go @@ -0,0 +1,40 @@ +package controller_test + +import ( + "testing" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/KubelanCloud/kks-provider-plugin/pkg/kloudlb" +) + +func TestIngressIPPrefersStatus(t *testing.T) { + svc := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{kloudlb.AnnotationIP: "172.173.200.1"}, + }, + Status: corev1.ServiceStatus{ + LoadBalancer: corev1.LoadBalancerStatus{ + Ingress: []corev1.LoadBalancerIngress{{IP: "172.173.200.2"}}, + }, + }, + } + + ip := ingressIP(svc) + if ip != "172.173.200.2" { + t.Fatalf("ingressIP = %q", ip) + } +} + +func ingressIP(svc *corev1.Service) string { + for _, ing := range svc.Status.LoadBalancer.Ingress { + if ing.IP != "" { + return ing.IP + } + } + if svc.Annotations != nil { + return svc.Annotations[kloudlb.AnnotationIP] + } + return "" +} diff --git a/pkg/kloudlb/speaker/speaker.go b/pkg/kloudlb/speaker/speaker.go new file mode 100644 index 0000000..d94813d --- /dev/null +++ b/pkg/kloudlb/speaker/speaker.go @@ -0,0 +1,345 @@ +package speaker + +import ( + "context" + "fmt" + "net" + "os" + "sync" + "time" + + corev1 "k8s.io/api/core/v1" + coordinationv1 "k8s.io/api/coordination/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/informers" + "k8s.io/client-go/kubernetes" + corelisters "k8s.io/client-go/listers/core/v1" + "k8s.io/client-go/tools/cache" + "k8s.io/client-go/util/workqueue" + + "github.com/KubelanCloud/kks-provider-plugin/pkg/kloudlb" + "github.com/vishvananda/netlink" +) + +type Speaker struct { + client kubernetes.Interface + nodeName string + ifaceName string + informer cache.SharedIndexInformer + lister corelisters.ServiceLister + queue workqueue.RateLimitingInterface + synced cache.InformerSynced + + mu sync.Mutex + leading map[string]string + boundIPs map[string]struct{} +} + +func New(client kubernetes.Interface, nodeName, ifaceName string) (*Speaker, error) { + if client == nil { + return nil, fmt.Errorf("kubernetes client is required") + } + if nodeName == "" { + return nil, fmt.Errorf("node name is required") + } + if ifaceName == "" { + ifaceName = "eth0" + } + + factory := informers.NewSharedInformerFactory(client, 30*time.Second) + informer := factory.Core().V1().Services().Informer() + queue := workqueue.NewRateLimitingQueue(workqueue.DefaultControllerRateLimiter()) + + s := &Speaker{ + client: client, + nodeName: nodeName, + ifaceName: ifaceName, + informer: informer, + lister: factory.Core().V1().Services().Lister(), + queue: queue, + synced: informer.HasSynced, + leading: make(map[string]string), + boundIPs: make(map[string]struct{}), + } + + informer.AddEventHandler(cache.ResourceEventHandlerFuncs{ + AddFunc: func(obj any) { s.enqueue(obj) }, + UpdateFunc: func(_, newObj any) { s.enqueue(newObj) }, + DeleteFunc: func(obj any) { s.enqueue(obj) }, + }) + + return s, nil +} + +func (s *Speaker) Run(ctx context.Context) error { + go s.informer.Run(ctx.Done()) + if !cache.WaitForCacheSync(ctx.Done(), s.synced) { + return fmt.Errorf("service informer cache sync failed") + } + + go wait.UntilWithContext(ctx, s.runWorker, time.Second) + <-ctx.Done() + s.releaseAll() + s.queue.ShutDown() + return nil +} + +func (s *Speaker) enqueue(obj any) { + svc, ok := obj.(*corev1.Service) + if !ok { + return + } + if svc.Spec.Type != corev1.ServiceTypeLoadBalancer { + return + } + key, err := cache.MetaNamespaceKeyFunc(obj) + if err != nil { + return + } + s.queue.Add(key) +} + +func (s *Speaker) runWorker(ctx context.Context) { + for s.processNext(ctx) { + } +} + +func (s *Speaker) processNext(ctx context.Context) bool { + item, shutdown := s.queue.Get() + if shutdown { + return false + } + defer s.queue.Done(item) + key, ok := item.(string) + if !ok { + s.queue.Forget(item) + return true + } + + if err := s.sync(ctx, key); err != nil { + s.queue.AddRateLimited(item) + return true + } + s.queue.Forget(item) + return true +} + +func (s *Speaker) sync(ctx context.Context, key string) error { + namespace, name, err := cache.SplitMetaNamespaceKey(key) + if err != nil { + return err + } + + svc, err := s.lister.Services(namespace).Get(name) + if apierrors.IsNotFound(err) { + s.stopLeading(key, "") + return nil + } + if err != nil { + return err + } + + ip := serviceExternalIP(svc) + if ip == "" || svc.DeletionTimestamp != nil { + s.stopLeading(key, ip) + return nil + } + + leaseName := leaseNameFor(key) + leader, err := s.acquireLease(ctx, leaseName, ip) + if err != nil { + return err + } + if leader != s.nodeName { + s.stopLeading(key, ip) + return nil + } + + return s.ensureVIP(ip) +} + +func serviceExternalIP(svc *corev1.Service) string { + for _, ing := range svc.Status.LoadBalancer.Ingress { + if ing.IP != "" { + return ing.IP + } + } + if svc.Annotations != nil { + return svc.Annotations[kloudlb.AnnotationIP] + } + return "" +} + +func leaseNameFor(serviceKey string) string { + return "kloud-lb-" + sanitizeLeaseName(serviceKey) +} + +func sanitizeLeaseName(value string) string { + out := make([]rune, 0, len(value)) + for _, r := range value { + switch { + case r >= 'a' && r <= 'z', r >= 'A' && r <= 'Z', r >= '0' && r <= '9', r == '-', r == '.': + out = append(out, r) + default: + out = append(out, '-') + } + } + name := string(out) + if len(name) > 52 { + name = name[:52] + } + return name +} + +func (s *Speaker) acquireLease(ctx context.Context, leaseName, vip string) (string, error) { + now := metav1.MicroTime{Time: time.Now()} + leaseClient := s.client.CoordinationV1().Leases(kloudlb.Namespace) + lease, err := leaseClient.Get(ctx, leaseName, metav1.GetOptions{}) + if apierrors.IsNotFound(err) { + lease = &coordinationv1.Lease{ + ObjectMeta: metav1.ObjectMeta{ + Name: leaseName, + Namespace: kloudlb.Namespace, + }, + Spec: coordinationv1.LeaseSpec{ + HolderIdentity: ptr(s.nodeName), + LeaseDurationSeconds: ptr(int32(15)), + AcquireTime: &now, + RenewTime: &now, + }, + } + if _, err := leaseClient.Create(ctx, lease, metav1.CreateOptions{}); err != nil { + return "", err + } + return s.nodeName, nil + } + if err != nil { + return "", err + } + + holder := "" + if lease.Spec.HolderIdentity != nil { + holder = *lease.Spec.HolderIdentity + } + if holder == "" || holder == s.nodeName || leaseExpired(lease) { + lease.Spec.HolderIdentity = ptr(s.nodeName) + lease.Spec.LeaseDurationSeconds = ptr(int32(15)) + lease.Spec.RenewTime = &now + if lease.Spec.AcquireTime == nil { + lease.Spec.AcquireTime = &now + } + if _, err := leaseClient.Update(ctx, lease, metav1.UpdateOptions{}); err != nil { + return "", err + } + return s.nodeName, nil + } + + _ = vip + return holder, nil +} + +func leaseExpired(lease *coordinationv1.Lease) bool { + if lease.Spec.RenewTime == nil || lease.Spec.LeaseDurationSeconds == nil { + return true + } + duration := time.Duration(*lease.Spec.LeaseDurationSeconds) * time.Second + return time.Since(lease.Spec.RenewTime.Time) > duration +} + +func (s *Speaker) ensureVIP(ip string) error { + s.mu.Lock() + defer s.mu.Unlock() + if _, ok := s.boundIPs[ip]; ok { + return nil + } + if err := addVIP(s.ifaceName, ip); err != nil { + return err + } + s.boundIPs[ip] = struct{}{} + return nil +} + +func (s *Speaker) stopLeading(serviceKey, ip string) { + s.mu.Lock() + defer s.mu.Unlock() + if ip != "" { + if _, ok := s.boundIPs[ip]; ok { + _ = removeVIP(s.ifaceName, ip) + delete(s.boundIPs, ip) + } + } + delete(s.leading, serviceKey) +} + +func (s *Speaker) releaseAll() { + s.mu.Lock() + defer s.mu.Unlock() + for ip := range s.boundIPs { + _ = removeVIP(s.ifaceName, ip) + } + s.boundIPs = make(map[string]struct{}) +} + +func addVIP(ifaceName, ip string) error { + link, err := netlink.LinkByName(ifaceName) + if err != nil { + return fmt.Errorf("lookup interface %s: %w", ifaceName, err) + } + parsed := net.ParseIP(ip) + if parsed == nil { + return fmt.Errorf("invalid vip %q", ip) + } + addr := &netlink.Addr{ + IPNet: &net.IPNet{IP: parsed.To4(), Mask: net.CIDRMask(32, 32)}, + } + if err := netlink.AddrAdd(link, addr); err != nil { + return fmt.Errorf("add vip %s on %s: %w", ip, ifaceName, err) + } + return sendGratuitousARP(link, parsed.To4()) +} + +func removeVIP(ifaceName, ip string) error { + link, err := netlink.LinkByName(ifaceName) + if err != nil { + return err + } + parsed := net.ParseIP(ip) + if parsed == nil { + return fmt.Errorf("invalid vip %q", ip) + } + addr := &netlink.Addr{ + IPNet: &net.IPNet{IP: parsed.To4(), Mask: net.CIDRMask(32, 32)}, + } + return netlink.AddrDel(link, addr) +} + +func sendGratuitousARP(link netlink.Link, ip net.IP) error { + if ip == nil { + return fmt.Errorf("ip is required") + } + _ = link + _ = ip + // VIP is bound locally; L2 peers learn the address when traffic flows. + return nil +} + +func ptr[T any](v T) *T { return &v } + +func NodeNameFromEnv() string { + if v := os.Getenv("NODE_NAME"); v != "" { + return v + } + if v, err := os.Hostname(); err == nil { + return v + } + return "" +} + +func InterfaceFromEnv() string { + if v := os.Getenv("KLOUD_LB_INTERFACE"); v != "" { + return v + } + return "eth0" +} diff --git a/pkg/lb/api/client.go b/pkg/lb/api/client.go new file mode 100644 index 0000000..f830e14 --- /dev/null +++ b/pkg/lb/api/client.go @@ -0,0 +1,127 @@ +package api + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "strings" + "time" + + "github.com/KubelanCloud/kks-provider-plugin/pkg/lb/provisioner" +) + +type ClientConfig struct { + BaseURL string + Token string + Timeout time.Duration +} + +type Client struct { + baseURL string + token string + client *http.Client +} + +func NewClient(cfg ClientConfig) *Client { + timeout := cfg.Timeout + if timeout <= 0 { + timeout = 30 * time.Second + } + return &Client{ + baseURL: strings.TrimRight(strings.TrimSpace(cfg.BaseURL), "/"), + token: strings.TrimSpace(cfg.Token), + client: &http.Client{Timeout: timeout}, + } +} + +func (c *Client) Allocate(ctx context.Context, req provisioner.AllocateRequest) (provisioner.LoadBalancer, error) { + var out provisioner.LoadBalancer + if err := c.doJSON(ctx, http.MethodPost, "/v1/loadbalancers", req, &out); err != nil { + return provisioner.LoadBalancer{}, err + } + return out, nil +} + +func (c *Client) Get(ctx context.Context, id string) (provisioner.LoadBalancer, error) { + var out provisioner.LoadBalancer + if err := c.doJSON(ctx, http.MethodGet, "/v1/loadbalancers/"+escapePath(id), nil, &out); err != nil { + return provisioner.LoadBalancer{}, err + } + return out, nil +} + +func (c *Client) Release(ctx context.Context, id string) error { + err := c.doJSON(ctx, http.MethodDelete, "/v1/loadbalancers/"+escapePath(id), nil, nil) + if isNotFound(err) { + return nil + } + return err +} + +func (c *Client) doJSON(ctx context.Context, method, path string, body any, out any) error { + var reader io.Reader + if body != nil { + raw, err := json.Marshal(body) + if err != nil { + return err + } + reader = bytes.NewReader(raw) + } + req, err := http.NewRequestWithContext(ctx, method, c.baseURL+path, reader) + if err != nil { + return err + } + if body != nil { + req.Header.Set("Content-Type", "application/json") + } + if c.token != "" { + req.Header.Set("Authorization", "Bearer "+c.token) + } + + resp, err := c.client.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + + raw, err := io.ReadAll(resp.Body) + if err != nil { + return err + } + if resp.StatusCode >= 400 { + message := strings.TrimSpace(string(raw)) + var payload struct { + Error string `json:"error"` + } + if json.Unmarshal(raw, &payload) == nil && payload.Error != "" { + message = payload.Error + } + if message == "" { + message = resp.Status + } + if resp.StatusCode == http.StatusNotFound { + return NewHTTPError(http.StatusNotFound, message) + } + return fmt.Errorf("lb api %s %s: %s", method, path, message) + } + if out == nil || len(raw) == 0 || resp.StatusCode == http.StatusNoContent { + return nil + } + if err := json.Unmarshal(raw, out); err != nil { + return fmt.Errorf("decode lb api response: %w", err) + } + return nil +} + +func escapePath(value string) string { + return strings.ReplaceAll(value, "/", "%2F") +} + +func isNotFound(err error) bool { + var httpErr *HTTPError + return errors.As(err, &httpErr) && httpErr.StatusCode == http.StatusNotFound +} diff --git a/pkg/lb/api/client_test.go b/pkg/lb/api/client_test.go new file mode 100644 index 0000000..279e9ab --- /dev/null +++ b/pkg/lb/api/client_test.go @@ -0,0 +1,39 @@ +package api + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + + "github.com/KubelanCloud/kks-provider-plugin/pkg/lb/provisioner" +) + +func TestClientAllocate(t *testing.T) { + t.Parallel() + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost || r.URL.Path != "/v1/loadbalancers" { + t.Fatalf("unexpected request: %s %s", r.Method, r.URL.Path) + } + w.WriteHeader(http.StatusCreated) + _ = json.NewEncoder(w).Encode(provisioner.LoadBalancer{ + ID: "abc", + IP: "172.173.200.1", + }) + })) + t.Cleanup(srv.Close) + + client := NewClient(ClientConfig{BaseURL: srv.URL}) + lb, err := client.Allocate(context.Background(), provisioner.AllocateRequest{ + Namespace: "default", + Name: "web", + }) + if err != nil { + t.Fatalf("Allocate failed: %v", err) + } + if lb.IP != "172.173.200.1" { + t.Fatalf("unexpected lb: %#v", lb) + } +} diff --git a/pkg/lb/api/errors.go b/pkg/lb/api/errors.go new file mode 100644 index 0000000..544a2af --- /dev/null +++ b/pkg/lb/api/errors.go @@ -0,0 +1,17 @@ +package api + +type HTTPError struct { + StatusCode int + Message string +} + +func (e *HTTPError) Error() string { + if e == nil { + return "" + } + return e.Message +} + +func NewHTTPError(status int, message string) error { + return &HTTPError{StatusCode: status, Message: message} +} diff --git a/pkg/lb/provisioner/types.go b/pkg/lb/provisioner/types.go new file mode 100644 index 0000000..0672f20 --- /dev/null +++ b/pkg/lb/provisioner/types.go @@ -0,0 +1,13 @@ +package provisioner + +type LoadBalancer struct { + ID string `json:"id"` + IP string `json:"ip"` + Namespace string `json:"namespace"` + Name string `json:"name"` +} + +type AllocateRequest struct { + Namespace string `json:"namespace"` + Name string `json:"name"` +}