use ownerReference control the lifecycle of user's kubeconfig and kubectl pod

Signed-off-by: Nhongming <talonwan@yunify.com>

cmd/controller-manager/app/controllers.go

@@ -124,7 +124,7 @@ func AddControllers(
 
 	csrController := certificatesigningrequest.NewController(client.Kubernetes(), kubernetesInformer, client.Config())
 
-	clusterRoleBindingController := clusterrolebinding.NewController(client.Kubernetes(), kubernetesInformer)
+	clusterRoleBindingController := clusterrolebinding.NewController(client.Kubernetes(), kubernetesInformer, kubesphereInformer)
 
 	clusterController := cluster.NewClusterController(
 		client.Kubernetes(),

pkg/constants/constants.go

@@ -38,7 +38,7 @@ const (
 	DisplayNameAnnotationKey       = "kubesphere.io/alias-name"
 	DescriptionAnnotationKey       = "kubesphere.io/description"
 	CreatorAnnotationKey           = "kubesphere.io/creator"
-	UsernameAnnotationKey          = "kubesphere.io/username"
+	UsernameLabelKey               = "kubesphere.io/username"
 	System                         = "system"
 	OpenPitrixRuntimeAnnotationKey = "openpitrix_runtime"
 	WorkspaceAdmin                 = "workspace-admin"

pkg/controller/certificatesigningrequest/certificatesigningrequest_controller.go

@@ -221,7 +221,7 @@ func (c *Controller) reconcile(key string) error {
 	}
 
 	// csr create by kubesphere auto approve
-	if username := csr.Annotations[constants.UsernameAnnotationKey]; username != "" {
+	if username := csr.Labels[constants.UsernameLabelKey]; username != "" {
 		err = c.Approve(csr)
 		if err != nil {
 			klog.Error(err)
@@ -280,7 +280,7 @@ func (c *Controller) Approve(csr *certificatesv1beta1.CertificateSigningRequest)
 }
 
 func (c *Controller) UpdateKubeconfig(csr *certificatesv1beta1.CertificateSigningRequest) error {
-	username := csr.Annotations[constants.UsernameAnnotationKey]
+	username := csr.Labels[constants.UsernameLabelKey]
 
 	err := c.kubeconfigOperator.UpdateKubeconfig(username, csr.Status.Certificate)
 

pkg/controller/clusterrolebinding/clusterrolebinding_controller.go

@@ -23,7 +23,7 @@ import (
 	"k8s.io/apimachinery/pkg/api/errors"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	"k8s.io/apimachinery/pkg/util/wait"
-	"k8s.io/client-go/informers"
+	k8sinformers "k8s.io/client-go/informers"
 	rbacv1informers "k8s.io/client-go/informers/rbac/v1"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/kubernetes/scheme"
@@ -34,6 +34,7 @@ import (
 	"k8s.io/client-go/util/workqueue"
 	"k8s.io/klog"
 	iamv1alpha2 "kubesphere.io/kubesphere/pkg/apis/iam/v1alpha2"
+	ksinformers "kubesphere.io/kubesphere/pkg/client/informers/externalversions"
 	"kubesphere.io/kubesphere/pkg/models/kubectl"
 	"time"
 )
@@ -63,7 +64,7 @@ type Controller struct {
 	kubectlOperator kubectl.Interface
 }
 
-func NewController(k8sClient kubernetes.Interface, informerFactory informers.SharedInformerFactory) *Controller {
+func NewController(k8sClient kubernetes.Interface, k8sInformer k8sinformers.SharedInformerFactory, ksInformer ksinformers.SharedInformerFactory) *Controller {
 	// Create event broadcaster
 	// Add sample-controller types to the default Kubernetes Scheme so Events can be
 	// logged for sample-controller types.
@@ -73,13 +74,13 @@ func NewController(k8sClient kubernetes.Interface, informerFactory informers.Sha
 	eventBroadcaster.StartLogging(klog.Infof)
 	eventBroadcaster.StartRecordingToSink(&typedcorev1.EventSinkImpl{Interface: k8sClient.CoreV1().Events("")})
 	recorder := eventBroadcaster.NewRecorder(scheme.Scheme, corev1.EventSource{Component: controllerName})
-	informer := informerFactory.Rbac().V1().ClusterRoleBindings()
+	informer := k8sInformer.Rbac().V1().ClusterRoleBindings()
 	ctl := &Controller{
 		k8sClient:       k8sClient,
 		informer:        informer,
 		lister:          informer.Lister(),
 		synced:          informer.Informer().HasSynced,
-		kubectlOperator: kubectl.NewOperator(k8sClient, informerFactory),
+		kubectlOperator: kubectl.NewOperator(k8sClient, k8sInformer, ksInformer),
 		workqueue:       workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "ClusterRoleBinding"),
 		recorder:        recorder,
 	}

pkg/kapis/resources/v1alpha2/handler.go

@@ -49,7 +49,8 @@ func newResourceHandler(k8sClient kubernetes.Interface, factory informers.Inform
 		gitVerifier:         git.NewGitVerifier(factory.KubernetesSharedInformerFactory()),
 		registryGetter:      registries.NewRegistryGetter(factory.KubernetesSharedInformerFactory()),
 		kubeconfigOperator:  kubeconfig.NewOperator(k8sClient, nil, masterURL),
-		kubectlOperator:     kubectl.NewOperator(k8sClient, factory.KubernetesSharedInformerFactory()),
+		kubectlOperator: kubectl.NewOperator(k8sClient, factory.KubernetesSharedInformerFactory(),
+			factory.KubeSphereSharedInformerFactory()),
 	}
 }
 

pkg/models/kubeconfig/kubeconfig.go

@@ -58,7 +58,6 @@ const (
 type Interface interface {
 	GetKubeConfig(username string) (string, error)
 	CreateKubeConfig(user *iamv1alpha2.User) error
-	DelKubeConfig(username string) error
 	UpdateKubeconfig(username string, certificate []byte) error
 }
 
@@ -135,7 +134,7 @@ func (o *operator) CreateKubeConfig(user *iamv1alpha2.User) error {
 	}
 
 	cm := &corev1.ConfigMap{TypeMeta: metav1.TypeMeta{Kind: configMapKind, APIVersion: configMapAPIVersion},
-		ObjectMeta: metav1.ObjectMeta{Name: configName, Annotations: map[string]string{constants.UsernameAnnotationKey: user.Name}},
+		ObjectMeta: metav1.ObjectMeta{Name: configName, Labels: map[string]string{constants.UsernameLabelKey: user.Name}},
 		Data:       map[string]string{kubeconfigFileName: string(kubeconfig)}}
 
 	err = controllerutil.SetControllerReference(user, cm, scheme.Scheme)
@@ -188,18 +187,6 @@ func (o *operator) GetKubeConfig(username string) (string, error) {
 	return string(data), nil
 }
 
-func (o *operator) DelKubeConfig(username string) error {
-	configName := fmt.Sprintf(kubeconfigNameFormat, username)
-
-	deletePolicy := metav1.DeletePropagationBackground
-	err := o.k8sclient.CoreV1().ConfigMaps(constants.KubeSphereControlNamespace).Delete(configName, &metav1.DeleteOptions{PropagationPolicy: &deletePolicy})
-	if err != nil {
-		klog.Errorln(err)
-		return err
-	}
-	return nil
-}
-
 func (o *operator) createCSR(username string) ([]byte, error) {
 	csrConfig := &certutil.Config{
 		CommonName:   username,
@@ -247,8 +234,8 @@ func (o *operator) createCSR(username string) ([]byte, error) {
 			APIVersion: "certificates.k8s.io/v1beta1",
 		},
 		ObjectMeta: metav1.ObjectMeta{
-			Name:        csrName,
-			Annotations: map[string]string{constants.UsernameAnnotationKey: username},
+			Name:   csrName,
+			Labels: map[string]string{constants.UsernameLabelKey: username},
 		},
 		Spec: certificatesv1beta1.CertificateSigningRequestSpec{
 			Request:  csr,

pkg/models/kubectl/kubectl.go

@@ -20,18 +20,20 @@ package kubectl
 
 import (
 	"fmt"
+	appsv1 "k8s.io/api/apps/v1"
+	"k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
-	"k8s.io/client-go/informers"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/labels"
+	k8sinformers "k8s.io/client-go/informers"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/klog"
+	"kubesphere.io/kubesphere/pkg/client/clientset/versioned/scheme"
+	ksinformers "kubesphere.io/kubesphere/pkg/client/informers/externalversions"
 	"kubesphere.io/kubesphere/pkg/models"
 	"math/rand"
 	"os"
-
-	appsv1 "k8s.io/api/apps/v1"
-	"k8s.io/api/core/v1"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apimachinery/pkg/labels"
+	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
 
 	"kubesphere.io/kubesphere/pkg/constants"
 )
@@ -44,16 +46,16 @@ const (
 type Interface interface {
 	GetKubectlPod(username string) (models.PodInfo, error)
 	CreateKubectlDeploy(username string) error
-	DeleteKubectlDeploy(username string) error
 }
 
 type operator struct {
-	k8sClient kubernetes.Interface
-	informers informers.SharedInformerFactory
+	k8sClient   kubernetes.Interface
+	k8sInformer k8sinformers.SharedInformerFactory
+	ksInformer  ksinformers.SharedInformerFactory
 }
 
-func NewOperator(k8sClient kubernetes.Interface, informers informers.SharedInformerFactory) Interface {
-	return &operator{k8sClient: k8sClient, informers: informers}
+func NewOperator(k8sClient kubernetes.Interface, k8sInformer k8sinformers.SharedInformerFactory, ksInformer ksinformers.SharedInformerFactory) Interface {
+	return &operator{k8sClient: k8sClient, k8sInformer: k8sInformer, ksInformer: ksInformer}
 }
 
 var DefaultImage = "kubesphere/kubectl:advanced-1.0.0"
@@ -66,7 +68,7 @@ func init() {
 
 func (o *operator) GetKubectlPod(username string) (models.PodInfo, error) {
 	deployName := fmt.Sprintf(deployNameFormat, username)
-	deploy, err := o.informers.Apps().V1().Deployments().Lister().Deployments(namespace).Get(deployName)
+	deploy, err := o.k8sInformer.Apps().V1().Deployments().Lister().Deployments(namespace).Get(deployName)
 	if err != nil {
 		klog.Errorln(err)
 		return models.PodInfo{}, err
@@ -74,7 +76,7 @@ func (o *operator) GetKubectlPod(username string) (models.PodInfo, error) {
 
 	selectors := deploy.Spec.Selector.MatchLabels
 	labelSelector := labels.Set(selectors).AsSelector()
-	pods, err := o.informers.Core().V1().Pods().Lister().Pods(namespace).List(labelSelector)
+	pods, err := o.k8sInformer.Core().V1().Pods().Lister().Pods(namespace).List(labelSelector)
 	if err != nil {
 		klog.Errorln(err)
 		return models.PodInfo{}, err
@@ -115,9 +117,20 @@ func selectCorrectPod(namespace string, pods []*v1.Pod) (kubectlPod *v1.Pod, err
 func (o *operator) CreateKubectlDeploy(username string) error {
 	deployName := fmt.Sprintf(deployNameFormat, username)
 
+	user, err := o.ksInformer.Iam().V1alpha2().Users().Lister().Get(username)
+
+	if err != nil {
+		klog.Error(err)
+		// ignore if user not exist
+		if errors.IsNotFound(err) {
+			return nil
+		}
+		return err
+	}
+
 	replica := int32(1)
-	selector := metav1.LabelSelector{MatchLabels: map[string]string{"username": username}}
-	deployment := appsv1.Deployment{
+	selector := metav1.LabelSelector{MatchLabels: map[string]string{constants.UsernameLabelKey: username}}
+	deployment := &appsv1.Deployment{
 		ObjectMeta: metav1.ObjectMeta{
 			Name: deployName,
 		},
@@ -127,7 +140,7 @@ func (o *operator) CreateKubectlDeploy(username string) error {
 			Template: v1.PodTemplateSpec{
 				ObjectMeta: metav1.ObjectMeta{
 					Labels: map[string]string{
-						"username": username,
+						constants.UsernameLabelKey: username,
 					},
 				},
 				Spec: v1.PodSpec{
@@ -142,25 +155,17 @@ func (o *operator) CreateKubectlDeploy(username string) error {
 		},
 	}
 
-	_, err := o.k8sClient.AppsV1().Deployments(namespace).Create(&deployment)
+	err = controllerutil.SetControllerReference(user, deployment, scheme.Scheme)
 
 	if err != nil {
-		if errors.IsAlreadyExists(err) {
-			return nil
-		}
-		klog.Error(err)
+		klog.Errorln(err)
 		return err
 	}
 
-	return nil
-}
+	_, err = o.k8sClient.AppsV1().Deployments(namespace).Create(deployment)
 
-func (o *operator) DeleteKubectlDeploy(username string) error {
-	deployName := fmt.Sprintf(deployNameFormat, username)
-
-	err := o.k8sClient.AppsV1().Deployments(namespace).Delete(deployName, metav1.NewDeleteOptions(0))
 	if err != nil {
-		if errors.IsNotFound(err) {
+		if errors.IsAlreadyExists(err) {
 			return nil
 		}
 		klog.Error(err)