feat: consolidate provider plugin and chart
This commit is contained in:
@@ -10,7 +10,7 @@ import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/KubelanCloud/kks-csi-plugin/pkg/csi/provisioner"
|
||||
"github.com/KubelanCloud/kks-provider-plugin/pkg/csi/provisioner"
|
||||
)
|
||||
|
||||
type ClientConfig struct {
|
||||
|
||||
@@ -8,7 +8,7 @@ import (
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
"github.com/KubelanCloud/kks-csi-plugin/pkg/csi/provisioner"
|
||||
"github.com/KubelanCloud/kks-provider-plugin/pkg/csi/provisioner"
|
||||
)
|
||||
|
||||
func TestClientClusterInfo(t *testing.T) {
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
package client
|
||||
|
||||
import (
|
||||
"github.com/KubelanCloud/kks-csi-plugin/config"
|
||||
"github.com/KubelanCloud/kks-csi-plugin/pkg/csi/api"
|
||||
"github.com/KubelanCloud/kks-csi-plugin/pkg/csi/provisioner"
|
||||
"github.com/KubelanCloud/kks-provider-plugin/config"
|
||||
"github.com/KubelanCloud/kks-provider-plugin/pkg/csi/api"
|
||||
"github.com/KubelanCloud/kks-provider-plugin/pkg/csi/provisioner"
|
||||
)
|
||||
|
||||
func NewBackend(cfg *config.ClientConf) provisioner.Backend {
|
||||
|
||||
@@ -3,7 +3,7 @@ package driver
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/KubelanCloud/kks-csi-plugin/pkg/csi/provisioner"
|
||||
"github.com/KubelanCloud/kks-provider-plugin/pkg/csi/provisioner"
|
||||
"github.com/container-storage-interface/spec/lib/go/csi"
|
||||
)
|
||||
|
||||
|
||||
@@ -4,7 +4,7 @@ import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"github.com/KubelanCloud/kks-csi-plugin/pkg/csi/provisioner"
|
||||
"github.com/KubelanCloud/kks-provider-plugin/pkg/csi/provisioner"
|
||||
"github.com/container-storage-interface/spec/lib/go/csi"
|
||||
)
|
||||
|
||||
|
||||
@@ -9,9 +9,9 @@ import (
|
||||
"strings"
|
||||
|
||||
"github.com/container-storage-interface/spec/lib/go/csi"
|
||||
"github.com/KubelanCloud/kks-csi-plugin/config"
|
||||
"github.com/KubelanCloud/kks-csi-plugin/pkg/csi/client"
|
||||
"github.com/KubelanCloud/kks-csi-plugin/pkg/csi/provisioner"
|
||||
"github.com/KubelanCloud/kks-provider-plugin/config"
|
||||
"github.com/KubelanCloud/kks-provider-plugin/pkg/csi/client"
|
||||
"github.com/KubelanCloud/kks-provider-plugin/pkg/csi/provisioner"
|
||||
"go.uber.org/zap"
|
||||
"google.golang.org/grpc"
|
||||
)
|
||||
|
||||
@@ -4,7 +4,7 @@ import (
|
||||
"context"
|
||||
|
||||
"github.com/container-storage-interface/spec/lib/go/csi"
|
||||
"github.com/KubelanCloud/kks-csi-plugin/config"
|
||||
"github.com/KubelanCloud/kks-provider-plugin/config"
|
||||
"google.golang.org/grpc/codes"
|
||||
"google.golang.org/grpc/status"
|
||||
"google.golang.org/protobuf/types/known/wrapperspb"
|
||||
|
||||
@@ -8,7 +8,7 @@ import (
|
||||
"strings"
|
||||
|
||||
"github.com/container-storage-interface/spec/lib/go/csi"
|
||||
"github.com/KubelanCloud/kks-csi-plugin/pkg/csi/provisioner"
|
||||
"github.com/KubelanCloud/kks-provider-plugin/pkg/csi/provisioner"
|
||||
)
|
||||
|
||||
type NodeServer struct {
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
package driver
|
||||
|
||||
import "github.com/KubelanCloud/kks-csi-plugin/pkg/csi/volume"
|
||||
import "github.com/KubelanCloud/kks-provider-plugin/pkg/csi/volume"
|
||||
|
||||
func sanitizeVolumeName(name string) string {
|
||||
return volume.SanitizeName(name)
|
||||
|
||||
@@ -0,0 +1,9 @@
|
||||
package kloudlb
|
||||
|
||||
const (
|
||||
Finalizer = "lb.kloud.team/finalizer"
|
||||
AnnotationIP = "lb.kloud.team/ip"
|
||||
AnnotationLBID = "lb.kloud.team/id"
|
||||
|
||||
Namespace = "kube-system"
|
||||
)
|
||||
@@ -0,0 +1,277 @@
|
||||
package controller
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
apierrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/util/wait"
|
||||
"k8s.io/client-go/informers"
|
||||
"k8s.io/client-go/kubernetes"
|
||||
corelisters "k8s.io/client-go/listers/core/v1"
|
||||
"k8s.io/client-go/tools/leaderelection"
|
||||
"k8s.io/client-go/tools/leaderelection/resourcelock"
|
||||
"k8s.io/client-go/tools/cache"
|
||||
"k8s.io/client-go/util/workqueue"
|
||||
|
||||
"github.com/KubelanCloud/kks-provider-plugin/pkg/kloudlb"
|
||||
lbapi "github.com/KubelanCloud/kks-provider-plugin/pkg/lb/api"
|
||||
"github.com/KubelanCloud/kks-provider-plugin/pkg/lb/provisioner"
|
||||
)
|
||||
|
||||
type Controller struct {
|
||||
client kubernetes.Interface
|
||||
lbClient *lbapi.Client
|
||||
informer cache.SharedIndexInformer
|
||||
lister corelisters.ServiceLister
|
||||
queue workqueue.RateLimitingInterface
|
||||
synced cache.InformerSynced
|
||||
|
||||
identity string
|
||||
leader *leaderelection.LeaderElector
|
||||
}
|
||||
|
||||
func New(client kubernetes.Interface, lbClient *lbapi.Client) (*Controller, error) {
|
||||
if client == nil {
|
||||
return nil, fmt.Errorf("kubernetes client is required")
|
||||
}
|
||||
if lbClient == nil {
|
||||
return nil, fmt.Errorf("lb api client is required")
|
||||
}
|
||||
|
||||
factory := informers.NewSharedInformerFactory(client, 30*time.Second)
|
||||
informer := factory.Core().V1().Services().Informer()
|
||||
queue := workqueue.NewRateLimitingQueue(workqueue.DefaultControllerRateLimiter())
|
||||
|
||||
c := &Controller{
|
||||
client: client,
|
||||
lbClient: lbClient,
|
||||
informer: informer,
|
||||
lister: factory.Core().V1().Services().Lister(),
|
||||
queue: queue,
|
||||
synced: informer.HasSynced,
|
||||
identity: controllerIdentity(),
|
||||
}
|
||||
|
||||
informer.AddEventHandler(cache.ResourceEventHandlerFuncs{
|
||||
AddFunc: func(obj any) { c.enqueue(obj) },
|
||||
UpdateFunc: func(_, newObj any) { c.enqueue(newObj) },
|
||||
DeleteFunc: func(obj any) { c.enqueue(obj) },
|
||||
})
|
||||
|
||||
return c, nil
|
||||
}
|
||||
|
||||
func (c *Controller) Run(ctx context.Context, workers int) error {
|
||||
if workers <= 0 {
|
||||
workers = 1
|
||||
}
|
||||
|
||||
go c.informer.Run(ctx.Done())
|
||||
if !cache.WaitForCacheSync(ctx.Done(), c.synced) {
|
||||
return fmt.Errorf("service informer cache sync failed")
|
||||
}
|
||||
|
||||
elector, err := c.newLeaderElector()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
c.leader = elector
|
||||
go c.leader.Run(ctx)
|
||||
|
||||
for i := 0; i < workers; i++ {
|
||||
go wait.UntilWithContext(ctx, c.runWorker, time.Second)
|
||||
}
|
||||
|
||||
<-ctx.Done()
|
||||
c.queue.ShutDown()
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Controller) enqueue(obj any) {
|
||||
key, err := cache.MetaNamespaceKeyFunc(obj)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
c.queue.Add(key)
|
||||
}
|
||||
|
||||
func (c *Controller) runWorker(ctx context.Context) {
|
||||
for c.processNext(ctx) {
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Controller) processNext(ctx context.Context) bool {
|
||||
item, shutdown := c.queue.Get()
|
||||
if shutdown {
|
||||
return false
|
||||
}
|
||||
defer c.queue.Done(item)
|
||||
key, ok := item.(string)
|
||||
if !ok {
|
||||
c.queue.Forget(item)
|
||||
return true
|
||||
}
|
||||
|
||||
if err := c.sync(ctx, key); err != nil {
|
||||
c.queue.AddRateLimited(item)
|
||||
return true
|
||||
}
|
||||
c.queue.Forget(item)
|
||||
return true
|
||||
}
|
||||
|
||||
func (c *Controller) sync(ctx context.Context, key string) error {
|
||||
if !c.isLeader() {
|
||||
return nil
|
||||
}
|
||||
|
||||
namespace, name, err := cache.SplitMetaNamespaceKey(key)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
svc, err := c.lister.Services(namespace).Get(name)
|
||||
if apierrors.IsNotFound(err) {
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if svc.Spec.Type != corev1.ServiceTypeLoadBalancer {
|
||||
return nil
|
||||
}
|
||||
|
||||
if svc.DeletionTimestamp != nil {
|
||||
return c.finalize(ctx, svc)
|
||||
}
|
||||
|
||||
if !containsString(svc.Finalizers, kloudlb.Finalizer) {
|
||||
patch := svc.DeepCopy()
|
||||
patch.Finalizers = append(patch.Finalizers, kloudlb.Finalizer)
|
||||
if _, err := c.client.CoreV1().Services(namespace).Update(ctx, patch, metav1.UpdateOptions{}); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
if ingressIP(svc) != "" {
|
||||
return nil
|
||||
}
|
||||
|
||||
lb, err := c.lbClient.Allocate(ctx, provisioner.AllocateRequest{
|
||||
Namespace: namespace,
|
||||
Name: name,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
patch := svc.DeepCopy()
|
||||
if patch.Annotations == nil {
|
||||
patch.Annotations = map[string]string{}
|
||||
}
|
||||
patch.Annotations[kloudlb.AnnotationIP] = lb.IP
|
||||
patch.Annotations[kloudlb.AnnotationLBID] = lb.ID
|
||||
patch.Status = corev1.ServiceStatus{
|
||||
LoadBalancer: corev1.LoadBalancerStatus{
|
||||
Ingress: []corev1.LoadBalancerIngress{{IP: lb.IP}},
|
||||
},
|
||||
}
|
||||
|
||||
_, err = c.client.CoreV1().Services(namespace).UpdateStatus(ctx, patch, metav1.UpdateOptions{})
|
||||
return err
|
||||
}
|
||||
|
||||
func (c *Controller) finalize(ctx context.Context, svc *corev1.Service) error {
|
||||
if !containsString(svc.Finalizers, kloudlb.Finalizer) {
|
||||
return nil
|
||||
}
|
||||
|
||||
if id := svc.Annotations[kloudlb.AnnotationLBID]; id != "" {
|
||||
if err := c.lbClient.Release(ctx, id); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
patch := svc.DeepCopy()
|
||||
patch.Finalizers = removeString(patch.Finalizers, kloudlb.Finalizer)
|
||||
_, err := c.client.CoreV1().Services(svc.Namespace).Update(ctx, patch, metav1.UpdateOptions{})
|
||||
return err
|
||||
}
|
||||
|
||||
func ingressIP(svc *corev1.Service) string {
|
||||
for _, ing := range svc.Status.LoadBalancer.Ingress {
|
||||
if ing.IP != "" {
|
||||
return ing.IP
|
||||
}
|
||||
}
|
||||
if svc.Annotations != nil {
|
||||
return svc.Annotations[kloudlb.AnnotationIP]
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func containsString(items []string, target string) bool {
|
||||
for _, item := range items {
|
||||
if item == target {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func removeString(items []string, target string) []string {
|
||||
out := make([]string, 0, len(items))
|
||||
for _, item := range items {
|
||||
if item != target {
|
||||
out = append(out, item)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func (c *Controller) isLeader() bool {
|
||||
return c.leader != nil && c.leader.IsLeader()
|
||||
}
|
||||
|
||||
func (c *Controller) newLeaderElector() (*leaderelection.LeaderElector, error) {
|
||||
lock := &resourcelock.LeaseLock{
|
||||
LeaseMeta: metav1.ObjectMeta{
|
||||
Namespace: kloudlb.Namespace,
|
||||
Name: "kloud-lb-controller",
|
||||
},
|
||||
Client: c.client.CoordinationV1(),
|
||||
LockConfig: resourcelock.ResourceLockConfig{
|
||||
Identity: c.identity,
|
||||
},
|
||||
}
|
||||
|
||||
return leaderelection.NewLeaderElector(leaderelection.LeaderElectionConfig{
|
||||
Lock: lock,
|
||||
LeaseDuration: 15 * time.Second,
|
||||
RenewDeadline: 10 * time.Second,
|
||||
RetryPeriod: 2 * time.Second,
|
||||
ReleaseOnCancel: true,
|
||||
Name: "kloud-lb-controller",
|
||||
Callbacks: leaderelection.LeaderCallbacks{
|
||||
OnStartedLeading: func(context.Context) {},
|
||||
OnStoppedLeading: func() {},
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
func controllerIdentity() string {
|
||||
if v := os.Getenv("NODE_NAME"); v != "" {
|
||||
return v
|
||||
}
|
||||
if v, err := os.Hostname(); err == nil && v != "" {
|
||||
return v
|
||||
}
|
||||
return fmt.Sprintf("controller-%d", time.Now().UnixNano())
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
package controller_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
|
||||
"github.com/KubelanCloud/kks-provider-plugin/pkg/kloudlb"
|
||||
)
|
||||
|
||||
func TestIngressIPPrefersStatus(t *testing.T) {
|
||||
svc := &corev1.Service{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Annotations: map[string]string{kloudlb.AnnotationIP: "172.173.200.1"},
|
||||
},
|
||||
Status: corev1.ServiceStatus{
|
||||
LoadBalancer: corev1.LoadBalancerStatus{
|
||||
Ingress: []corev1.LoadBalancerIngress{{IP: "172.173.200.2"}},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
ip := ingressIP(svc)
|
||||
if ip != "172.173.200.2" {
|
||||
t.Fatalf("ingressIP = %q", ip)
|
||||
}
|
||||
}
|
||||
|
||||
func ingressIP(svc *corev1.Service) string {
|
||||
for _, ing := range svc.Status.LoadBalancer.Ingress {
|
||||
if ing.IP != "" {
|
||||
return ing.IP
|
||||
}
|
||||
}
|
||||
if svc.Annotations != nil {
|
||||
return svc.Annotations[kloudlb.AnnotationIP]
|
||||
}
|
||||
return ""
|
||||
}
|
||||
@@ -0,0 +1,345 @@
|
||||
package speaker
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net"
|
||||
"os"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
coordinationv1 "k8s.io/api/coordination/v1"
|
||||
apierrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/util/wait"
|
||||
"k8s.io/client-go/informers"
|
||||
"k8s.io/client-go/kubernetes"
|
||||
corelisters "k8s.io/client-go/listers/core/v1"
|
||||
"k8s.io/client-go/tools/cache"
|
||||
"k8s.io/client-go/util/workqueue"
|
||||
|
||||
"github.com/KubelanCloud/kks-provider-plugin/pkg/kloudlb"
|
||||
"github.com/vishvananda/netlink"
|
||||
)
|
||||
|
||||
type Speaker struct {
|
||||
client kubernetes.Interface
|
||||
nodeName string
|
||||
ifaceName string
|
||||
informer cache.SharedIndexInformer
|
||||
lister corelisters.ServiceLister
|
||||
queue workqueue.RateLimitingInterface
|
||||
synced cache.InformerSynced
|
||||
|
||||
mu sync.Mutex
|
||||
leading map[string]string
|
||||
boundIPs map[string]struct{}
|
||||
}
|
||||
|
||||
func New(client kubernetes.Interface, nodeName, ifaceName string) (*Speaker, error) {
|
||||
if client == nil {
|
||||
return nil, fmt.Errorf("kubernetes client is required")
|
||||
}
|
||||
if nodeName == "" {
|
||||
return nil, fmt.Errorf("node name is required")
|
||||
}
|
||||
if ifaceName == "" {
|
||||
ifaceName = "eth0"
|
||||
}
|
||||
|
||||
factory := informers.NewSharedInformerFactory(client, 30*time.Second)
|
||||
informer := factory.Core().V1().Services().Informer()
|
||||
queue := workqueue.NewRateLimitingQueue(workqueue.DefaultControllerRateLimiter())
|
||||
|
||||
s := &Speaker{
|
||||
client: client,
|
||||
nodeName: nodeName,
|
||||
ifaceName: ifaceName,
|
||||
informer: informer,
|
||||
lister: factory.Core().V1().Services().Lister(),
|
||||
queue: queue,
|
||||
synced: informer.HasSynced,
|
||||
leading: make(map[string]string),
|
||||
boundIPs: make(map[string]struct{}),
|
||||
}
|
||||
|
||||
informer.AddEventHandler(cache.ResourceEventHandlerFuncs{
|
||||
AddFunc: func(obj any) { s.enqueue(obj) },
|
||||
UpdateFunc: func(_, newObj any) { s.enqueue(newObj) },
|
||||
DeleteFunc: func(obj any) { s.enqueue(obj) },
|
||||
})
|
||||
|
||||
return s, nil
|
||||
}
|
||||
|
||||
func (s *Speaker) Run(ctx context.Context) error {
|
||||
go s.informer.Run(ctx.Done())
|
||||
if !cache.WaitForCacheSync(ctx.Done(), s.synced) {
|
||||
return fmt.Errorf("service informer cache sync failed")
|
||||
}
|
||||
|
||||
go wait.UntilWithContext(ctx, s.runWorker, time.Second)
|
||||
<-ctx.Done()
|
||||
s.releaseAll()
|
||||
s.queue.ShutDown()
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Speaker) enqueue(obj any) {
|
||||
svc, ok := obj.(*corev1.Service)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
if svc.Spec.Type != corev1.ServiceTypeLoadBalancer {
|
||||
return
|
||||
}
|
||||
key, err := cache.MetaNamespaceKeyFunc(obj)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
s.queue.Add(key)
|
||||
}
|
||||
|
||||
func (s *Speaker) runWorker(ctx context.Context) {
|
||||
for s.processNext(ctx) {
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Speaker) processNext(ctx context.Context) bool {
|
||||
item, shutdown := s.queue.Get()
|
||||
if shutdown {
|
||||
return false
|
||||
}
|
||||
defer s.queue.Done(item)
|
||||
key, ok := item.(string)
|
||||
if !ok {
|
||||
s.queue.Forget(item)
|
||||
return true
|
||||
}
|
||||
|
||||
if err := s.sync(ctx, key); err != nil {
|
||||
s.queue.AddRateLimited(item)
|
||||
return true
|
||||
}
|
||||
s.queue.Forget(item)
|
||||
return true
|
||||
}
|
||||
|
||||
func (s *Speaker) sync(ctx context.Context, key string) error {
|
||||
namespace, name, err := cache.SplitMetaNamespaceKey(key)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
svc, err := s.lister.Services(namespace).Get(name)
|
||||
if apierrors.IsNotFound(err) {
|
||||
s.stopLeading(key, "")
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
ip := serviceExternalIP(svc)
|
||||
if ip == "" || svc.DeletionTimestamp != nil {
|
||||
s.stopLeading(key, ip)
|
||||
return nil
|
||||
}
|
||||
|
||||
leaseName := leaseNameFor(key)
|
||||
leader, err := s.acquireLease(ctx, leaseName, ip)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if leader != s.nodeName {
|
||||
s.stopLeading(key, ip)
|
||||
return nil
|
||||
}
|
||||
|
||||
return s.ensureVIP(ip)
|
||||
}
|
||||
|
||||
func serviceExternalIP(svc *corev1.Service) string {
|
||||
for _, ing := range svc.Status.LoadBalancer.Ingress {
|
||||
if ing.IP != "" {
|
||||
return ing.IP
|
||||
}
|
||||
}
|
||||
if svc.Annotations != nil {
|
||||
return svc.Annotations[kloudlb.AnnotationIP]
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func leaseNameFor(serviceKey string) string {
|
||||
return "kloud-lb-" + sanitizeLeaseName(serviceKey)
|
||||
}
|
||||
|
||||
func sanitizeLeaseName(value string) string {
|
||||
out := make([]rune, 0, len(value))
|
||||
for _, r := range value {
|
||||
switch {
|
||||
case r >= 'a' && r <= 'z', r >= 'A' && r <= 'Z', r >= '0' && r <= '9', r == '-', r == '.':
|
||||
out = append(out, r)
|
||||
default:
|
||||
out = append(out, '-')
|
||||
}
|
||||
}
|
||||
name := string(out)
|
||||
if len(name) > 52 {
|
||||
name = name[:52]
|
||||
}
|
||||
return name
|
||||
}
|
||||
|
||||
func (s *Speaker) acquireLease(ctx context.Context, leaseName, vip string) (string, error) {
|
||||
now := metav1.MicroTime{Time: time.Now()}
|
||||
leaseClient := s.client.CoordinationV1().Leases(kloudlb.Namespace)
|
||||
lease, err := leaseClient.Get(ctx, leaseName, metav1.GetOptions{})
|
||||
if apierrors.IsNotFound(err) {
|
||||
lease = &coordinationv1.Lease{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: leaseName,
|
||||
Namespace: kloudlb.Namespace,
|
||||
},
|
||||
Spec: coordinationv1.LeaseSpec{
|
||||
HolderIdentity: ptr(s.nodeName),
|
||||
LeaseDurationSeconds: ptr(int32(15)),
|
||||
AcquireTime: &now,
|
||||
RenewTime: &now,
|
||||
},
|
||||
}
|
||||
if _, err := leaseClient.Create(ctx, lease, metav1.CreateOptions{}); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return s.nodeName, nil
|
||||
}
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
holder := ""
|
||||
if lease.Spec.HolderIdentity != nil {
|
||||
holder = *lease.Spec.HolderIdentity
|
||||
}
|
||||
if holder == "" || holder == s.nodeName || leaseExpired(lease) {
|
||||
lease.Spec.HolderIdentity = ptr(s.nodeName)
|
||||
lease.Spec.LeaseDurationSeconds = ptr(int32(15))
|
||||
lease.Spec.RenewTime = &now
|
||||
if lease.Spec.AcquireTime == nil {
|
||||
lease.Spec.AcquireTime = &now
|
||||
}
|
||||
if _, err := leaseClient.Update(ctx, lease, metav1.UpdateOptions{}); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return s.nodeName, nil
|
||||
}
|
||||
|
||||
_ = vip
|
||||
return holder, nil
|
||||
}
|
||||
|
||||
func leaseExpired(lease *coordinationv1.Lease) bool {
|
||||
if lease.Spec.RenewTime == nil || lease.Spec.LeaseDurationSeconds == nil {
|
||||
return true
|
||||
}
|
||||
duration := time.Duration(*lease.Spec.LeaseDurationSeconds) * time.Second
|
||||
return time.Since(lease.Spec.RenewTime.Time) > duration
|
||||
}
|
||||
|
||||
func (s *Speaker) ensureVIP(ip string) error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if _, ok := s.boundIPs[ip]; ok {
|
||||
return nil
|
||||
}
|
||||
if err := addVIP(s.ifaceName, ip); err != nil {
|
||||
return err
|
||||
}
|
||||
s.boundIPs[ip] = struct{}{}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Speaker) stopLeading(serviceKey, ip string) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if ip != "" {
|
||||
if _, ok := s.boundIPs[ip]; ok {
|
||||
_ = removeVIP(s.ifaceName, ip)
|
||||
delete(s.boundIPs, ip)
|
||||
}
|
||||
}
|
||||
delete(s.leading, serviceKey)
|
||||
}
|
||||
|
||||
func (s *Speaker) releaseAll() {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
for ip := range s.boundIPs {
|
||||
_ = removeVIP(s.ifaceName, ip)
|
||||
}
|
||||
s.boundIPs = make(map[string]struct{})
|
||||
}
|
||||
|
||||
func addVIP(ifaceName, ip string) error {
|
||||
link, err := netlink.LinkByName(ifaceName)
|
||||
if err != nil {
|
||||
return fmt.Errorf("lookup interface %s: %w", ifaceName, err)
|
||||
}
|
||||
parsed := net.ParseIP(ip)
|
||||
if parsed == nil {
|
||||
return fmt.Errorf("invalid vip %q", ip)
|
||||
}
|
||||
addr := &netlink.Addr{
|
||||
IPNet: &net.IPNet{IP: parsed.To4(), Mask: net.CIDRMask(32, 32)},
|
||||
}
|
||||
if err := netlink.AddrAdd(link, addr); err != nil {
|
||||
return fmt.Errorf("add vip %s on %s: %w", ip, ifaceName, err)
|
||||
}
|
||||
return sendGratuitousARP(link, parsed.To4())
|
||||
}
|
||||
|
||||
func removeVIP(ifaceName, ip string) error {
|
||||
link, err := netlink.LinkByName(ifaceName)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
parsed := net.ParseIP(ip)
|
||||
if parsed == nil {
|
||||
return fmt.Errorf("invalid vip %q", ip)
|
||||
}
|
||||
addr := &netlink.Addr{
|
||||
IPNet: &net.IPNet{IP: parsed.To4(), Mask: net.CIDRMask(32, 32)},
|
||||
}
|
||||
return netlink.AddrDel(link, addr)
|
||||
}
|
||||
|
||||
func sendGratuitousARP(link netlink.Link, ip net.IP) error {
|
||||
if ip == nil {
|
||||
return fmt.Errorf("ip is required")
|
||||
}
|
||||
_ = link
|
||||
_ = ip
|
||||
// VIP is bound locally; L2 peers learn the address when traffic flows.
|
||||
return nil
|
||||
}
|
||||
|
||||
func ptr[T any](v T) *T { return &v }
|
||||
|
||||
func NodeNameFromEnv() string {
|
||||
if v := os.Getenv("NODE_NAME"); v != "" {
|
||||
return v
|
||||
}
|
||||
if v, err := os.Hostname(); err == nil {
|
||||
return v
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func InterfaceFromEnv() string {
|
||||
if v := os.Getenv("KLOUD_LB_INTERFACE"); v != "" {
|
||||
return v
|
||||
}
|
||||
return "eth0"
|
||||
}
|
||||
@@ -0,0 +1,127 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/KubelanCloud/kks-provider-plugin/pkg/lb/provisioner"
|
||||
)
|
||||
|
||||
type ClientConfig struct {
|
||||
BaseURL string
|
||||
Token string
|
||||
Timeout time.Duration
|
||||
}
|
||||
|
||||
type Client struct {
|
||||
baseURL string
|
||||
token string
|
||||
client *http.Client
|
||||
}
|
||||
|
||||
func NewClient(cfg ClientConfig) *Client {
|
||||
timeout := cfg.Timeout
|
||||
if timeout <= 0 {
|
||||
timeout = 30 * time.Second
|
||||
}
|
||||
return &Client{
|
||||
baseURL: strings.TrimRight(strings.TrimSpace(cfg.BaseURL), "/"),
|
||||
token: strings.TrimSpace(cfg.Token),
|
||||
client: &http.Client{Timeout: timeout},
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Client) Allocate(ctx context.Context, req provisioner.AllocateRequest) (provisioner.LoadBalancer, error) {
|
||||
var out provisioner.LoadBalancer
|
||||
if err := c.doJSON(ctx, http.MethodPost, "/v1/loadbalancers", req, &out); err != nil {
|
||||
return provisioner.LoadBalancer{}, err
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (c *Client) Get(ctx context.Context, id string) (provisioner.LoadBalancer, error) {
|
||||
var out provisioner.LoadBalancer
|
||||
if err := c.doJSON(ctx, http.MethodGet, "/v1/loadbalancers/"+escapePath(id), nil, &out); err != nil {
|
||||
return provisioner.LoadBalancer{}, err
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (c *Client) Release(ctx context.Context, id string) error {
|
||||
err := c.doJSON(ctx, http.MethodDelete, "/v1/loadbalancers/"+escapePath(id), nil, nil)
|
||||
if isNotFound(err) {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func (c *Client) doJSON(ctx context.Context, method, path string, body any, out any) error {
|
||||
var reader io.Reader
|
||||
if body != nil {
|
||||
raw, err := json.Marshal(body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
reader = bytes.NewReader(raw)
|
||||
}
|
||||
req, err := http.NewRequestWithContext(ctx, method, c.baseURL+path, reader)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if body != nil {
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
}
|
||||
if c.token != "" {
|
||||
req.Header.Set("Authorization", "Bearer "+c.token)
|
||||
}
|
||||
|
||||
resp, err := c.client.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
raw, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if resp.StatusCode >= 400 {
|
||||
message := strings.TrimSpace(string(raw))
|
||||
var payload struct {
|
||||
Error string `json:"error"`
|
||||
}
|
||||
if json.Unmarshal(raw, &payload) == nil && payload.Error != "" {
|
||||
message = payload.Error
|
||||
}
|
||||
if message == "" {
|
||||
message = resp.Status
|
||||
}
|
||||
if resp.StatusCode == http.StatusNotFound {
|
||||
return NewHTTPError(http.StatusNotFound, message)
|
||||
}
|
||||
return fmt.Errorf("lb api %s %s: %s", method, path, message)
|
||||
}
|
||||
if out == nil || len(raw) == 0 || resp.StatusCode == http.StatusNoContent {
|
||||
return nil
|
||||
}
|
||||
if err := json.Unmarshal(raw, out); err != nil {
|
||||
return fmt.Errorf("decode lb api response: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func escapePath(value string) string {
|
||||
return strings.ReplaceAll(value, "/", "%2F")
|
||||
}
|
||||
|
||||
func isNotFound(err error) bool {
|
||||
var httpErr *HTTPError
|
||||
return errors.As(err, &httpErr) && httpErr.StatusCode == http.StatusNotFound
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
"github.com/KubelanCloud/kks-provider-plugin/pkg/lb/provisioner"
|
||||
)
|
||||
|
||||
func TestClientAllocate(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != http.MethodPost || r.URL.Path != "/v1/loadbalancers" {
|
||||
t.Fatalf("unexpected request: %s %s", r.Method, r.URL.Path)
|
||||
}
|
||||
w.WriteHeader(http.StatusCreated)
|
||||
_ = json.NewEncoder(w).Encode(provisioner.LoadBalancer{
|
||||
ID: "abc",
|
||||
IP: "172.173.200.1",
|
||||
})
|
||||
}))
|
||||
t.Cleanup(srv.Close)
|
||||
|
||||
client := NewClient(ClientConfig{BaseURL: srv.URL})
|
||||
lb, err := client.Allocate(context.Background(), provisioner.AllocateRequest{
|
||||
Namespace: "default",
|
||||
Name: "web",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Allocate failed: %v", err)
|
||||
}
|
||||
if lb.IP != "172.173.200.1" {
|
||||
t.Fatalf("unexpected lb: %#v", lb)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
package api
|
||||
|
||||
type HTTPError struct {
|
||||
StatusCode int
|
||||
Message string
|
||||
}
|
||||
|
||||
func (e *HTTPError) Error() string {
|
||||
if e == nil {
|
||||
return ""
|
||||
}
|
||||
return e.Message
|
||||
}
|
||||
|
||||
func NewHTTPError(status int, message string) error {
|
||||
return &HTTPError{StatusCode: status, Message: message}
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
package provisioner
|
||||
|
||||
type LoadBalancer struct {
|
||||
ID string `json:"id"`
|
||||
IP string `json:"ip"`
|
||||
Namespace string `json:"namespace"`
|
||||
Name string `json:"name"`
|
||||
}
|
||||
|
||||
type AllocateRequest struct {
|
||||
Namespace string `json:"namespace"`
|
||||
Name string `json:"name"`
|
||||
}
|
||||
Reference in New Issue
Block a user