fix: runtime release

Signed-off-by: Nhongming <talonwan@yunify.com>

pkg/apis/iam/v1alpha2/register.go

@@ -20,11 +20,14 @@ package v1alpha2
 import (
 	"github.com/emicklei/go-restful"
 	"github.com/emicklei/go-restful-openapi"
+	rbacv1 "k8s.io/api/rbac/v1"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"kubesphere.io/kubesphere/pkg/apiserver/iam"
 	"kubesphere.io/kubesphere/pkg/apiserver/runtime"
 	"kubesphere.io/kubesphere/pkg/errors"
 	"kubesphere.io/kubesphere/pkg/models"
+	"kubesphere.io/kubesphere/pkg/models/iam/policy"
+	"net/http"
 )
 
 const GroupName = "iam.kubesphere.io"
@@ -40,181 +43,220 @@ func addWebService(c *restful.Container) error {
 	tags := []string{"IAM"}
 	ws := runtime.NewWebService(GroupVersion)
 
+	ok := "ok"
+	pageableUserList := struct {
+		Items      []models.User `json:"items"`
+		TotalCount int           `json:"total_count"`
+	}{}
+
 	ws.Route(ws.POST("/authenticate").
 		To(iam.TokenReviewHandler).
-		Doc("Token review").
+		Doc("TokenReview attempts to authenticate a token to a known user. Note: TokenReview requests may be cached by the webhook token authenticator plugin in the kube-apiserver.").
 		Reads(iam.TokenReview{}).
-		Writes(iam.TokenReview{}).
-		Doc("k8s token review").
+		Returns(http.StatusOK, ok, iam.TokenReview{}).
 		Metadata(restfulspec.KeyOpenAPITags, tags))
 	ws.Route(ws.POST("/login").
 		To(iam.LoginHandler).
-		Doc("User login").
+		Doc("KubeSphere APIs support token-based authentication via the Authtoken request header. The POST Login API is used to retrieve the authentication token. After the authentication token is obtained, it must be inserted into the Authtoken header for all requests.").
 		Reads(iam.LoginRequest{}).
-		Writes(models.Token{}).
+		Returns(http.StatusOK, ok, models.Token{}).
 		Metadata(restfulspec.KeyOpenAPITags, tags))
 	ws.Route(ws.GET("/users/{username}").
 		To(iam.DescribeUser).
-		Doc("User detail").
+		Doc("Describes the specified user.").
 		Param(ws.PathParameter("username", "username")).
-		Writes(models.User{}).
+		Returns(http.StatusOK, ok, models.User{}).
 		Metadata(restfulspec.KeyOpenAPITags, tags))
 	ws.Route(ws.POST("/users").
 		To(iam.CreateUser).
+		Doc("Create a user account.").
 		Reads(models.User{}).
-		Writes(errors.Error{}).
-		Doc("Create user").
+		Returns(http.StatusOK, ok, errors.Error{}).
 		Metadata(restfulspec.KeyOpenAPITags, tags))
 	ws.Route(ws.DELETE("/users/{name}").
 		To(iam.DeleteUser).
+		Doc("Remove a specified user.").
 		Param(ws.PathParameter("name", "username")).
-		Doc("Delete user").
-		Writes(errors.Error{}).
+		Returns(http.StatusOK, ok, errors.Error{}).
 		Metadata(restfulspec.KeyOpenAPITags, tags))
 	ws.Route(ws.PUT("/users/{name}").
 		To(iam.UpdateUser).
+		Doc("Updates information about the specified user.").
 		Param(ws.PathParameter("name", "username")).
 		Reads(models.User{}).
-		Writes(errors.Error{}).
-		Doc("Update user").
+		Returns(http.StatusOK, ok, errors.Error{}).
 		Metadata(restfulspec.KeyOpenAPITags, tags))
-
 	ws.Route(ws.GET("/users/{name}/log").
 		To(iam.UserLoginLog).
+		Doc("This method is used to retrieve the \"login logs\" for the specified user.").
 		Param(ws.PathParameter("name", "username")).
-		Doc("User login log").
-		Writes([]map[string]string{}).
+		Returns(http.StatusOK, ok, struct {
+			LoginTime string `json:"login_time"`
+			LoginIP   string `json:"login_ip"`
+		}{}).
 		Metadata(restfulspec.KeyOpenAPITags, tags))
 	ws.Route(ws.GET("/users").
 		To(iam.ListUsers).
-		Doc("User list").
-		Writes(models.PageableResponse{}).
+		Doc("List all users.").
+		Returns(http.StatusOK, ok, pageableUserList).
 		Metadata(restfulspec.KeyOpenAPITags, tags))
 	ws.Route(ws.GET("/groups").
 		To(iam.ListGroups).
-		Writes([]models.Group{}).
-		Doc("User group list").
+		Doc("List all user groups.").
+		Returns(http.StatusOK, ok, []models.Group{}).
 		Metadata(restfulspec.KeyOpenAPITags, tags))
-
 	ws.Route(ws.GET("/groups/{path}").
 		To(iam.DescribeGroup).
-		Param(ws.PathParameter("path", "group path")).
-		Doc("User group detail").
+		Doc("Describes the specified user group.").
+		Param(ws.PathParameter("path", "user group path separated by colon.")).
+		Returns(http.StatusOK, ok, models.Group{}).
 		Metadata(restfulspec.KeyOpenAPITags, tags))
 	ws.Route(ws.GET("/groups/{path}/users").
 		To(iam.ListGroupUsers).
-		Param(ws.PathParameter("path", "group path")).
-		Doc("Group user list").
+		Doc("List all users in the specified user group.").
+		Param(ws.PathParameter("path", "user group path separated by colon.")).
+		Returns(http.StatusOK, ok, []models.User{}).
 		Metadata(restfulspec.KeyOpenAPITags, tags))
 	ws.Route(ws.POST("/groups").
 		To(iam.CreateGroup).
+		Doc("Create a user group.").
 		Reads(models.Group{}).
-		Doc("Create user group").
+		Returns(http.StatusOK, ok, models.Group{}).
 		Metadata(restfulspec.KeyOpenAPITags, tags))
 	ws.Route(ws.DELETE("/groups/{path}").
 		To(iam.DeleteGroup).
-		Param(ws.PathParameter("path", "group path")).
-		Doc("Delete user group").
+		Doc("Delete a user group.").
+		Param(ws.PathParameter("path", "user group path separated by colon.")).
+		Returns(http.StatusOK, ok, errors.Error{}).
 		Metadata(restfulspec.KeyOpenAPITags, tags))
 	ws.Route(ws.PUT("/groups/{path}").
 		To(iam.UpdateGroup).
-		Param(ws.PathParameter("path", "group path")).
-		Doc("Update user group").
+		Doc("Updates information about the user group.").
+		Param(ws.PathParameter("path", "user group path separated by colon.")).
+		Reads(models.Group{}).
+		Returns(http.StatusOK, ok, models.Group{}).
 		Metadata(restfulspec.KeyOpenAPITags, tags))
 	ws.Route(ws.GET("/users/{username}/roles").
 		To(iam.ListUserRoles).
+		Doc("This method is used to retrieve all the roles that are assigned to the user.").
 		Param(ws.PathParameter("username", "username")).
-		Doc("Get user role list").
+		Returns(http.StatusOK, ok, iam.RoleList{}).
 		Metadata(restfulspec.KeyOpenAPITags, tags))
 	ws.Route(ws.GET("/namespaces/{namespace}/roles").
 		To(iam.ListRoles).
-		Param(ws.PathParameter("namespace", "namespace")).
-		Doc("Get role list").
+		Doc("This method is used to retrieve the roles that are assigned to the user in the specified namespace.").
+		Param(ws.PathParameter("namespace", "kubernetes namespace")).
+		Returns(http.StatusOK, ok, struct {
+			Items      []rbacv1.Role `json:"items"`
+			TotalCount int           `json:"total_count"`
+		}{}).
 		Metadata(restfulspec.KeyOpenAPITags, tags))
 	ws.Route(ws.GET("/clusterroles").
 		To(iam.ListClusterRoles).
-		Doc("Get cluster role list").
+		Doc("List all cluster roles.").
+		Returns(http.StatusOK, ok, struct {
+			Items      []rbacv1.ClusterRole `json:"items"`
+			TotalCount int                  `json:"total_count"`
+		}{}).
 		Metadata(restfulspec.KeyOpenAPITags, tags))
 	ws.Route(ws.GET("/namespaces/{namespace}/roles/{role}/users").
 		To(iam.ListRoleUsers).
-		Param(ws.PathParameter("namespace", "namespace")).
+		Doc("This method is used to retrieve the users that are bind the role in the specified namespace.").
+		Param(ws.PathParameter("namespace", "kubernetes namespace")).
 		Param(ws.PathParameter("role", "role name")).
-		Doc("Get user list by role").
+		Returns(http.StatusOK, ok, []models.User{}).
 		Metadata(restfulspec.KeyOpenAPITags, tags))
 	ws.Route(ws.GET("/namespaces/{namespace}/users").
 		To(iam.ListNamespaceUsers).
-		Param(ws.PathParameter("namespace", "namespace")).
-		Doc("Get user list by namespace").
+		Doc("List all users in the specified namespace").
+		Param(ws.PathParameter("namespace", "kubernetes namespace")).
+		Returns(http.StatusOK, ok, []models.User{}).
 		Metadata(restfulspec.KeyOpenAPITags, tags))
 	ws.Route(ws.GET("/clusterroles/{clusterrole}/users").
 		To(iam.ListClusterRoleUsers).
+		Doc("List all users that are bind the cluster role.").
 		Param(ws.PathParameter("clusterrole", "cluster role name")).
-		Doc("Get user list by cluster role").
+		Returns(http.StatusOK, ok, pageableUserList).
 		Metadata(restfulspec.KeyOpenAPITags, tags))
 	ws.Route(ws.GET("/clusterroles/{clusterrole}/rules").
 		To(iam.ListClusterRoleRules).
+		Doc("List all policy rules of the specified cluster role.").
 		Param(ws.PathParameter("clusterrole", "cluster role name")).
-		Doc("Get cluster role detail").
+		Returns(http.StatusOK, ok, []models.SimpleRule{}).
+		Metadata(restfulspec.KeyOpenAPITags, tags))
+	ws.Route(ws.GET("/namespaces/{namespace}/roles/{role}/rules").
+		To(iam.ListRoleRules).
+		Doc("List all policy rules of the specified role.").
+		Param(ws.PathParameter("namespace", "kubernetes namespace")).
+		Param(ws.PathParameter("role", "role name")).
+		Returns(http.StatusOK, ok, []models.SimpleRule{}).
+		Metadata(restfulspec.KeyOpenAPITags, tags))
+	ws.Route(ws.GET("/devops/{devops}/roles/{role}/rules").
+		To(iam.ListDevopsRoleRules).
+		Doc("List all policy rules of the specified role.").
+		Param(ws.PathParameter("devops", "devops project id")).
+		Param(ws.PathParameter("role", "devops role name")).
+		Returns(http.StatusOK, ok, []models.SimpleRule{}).
 		Metadata(restfulspec.KeyOpenAPITags, tags))
 	ws.Route(ws.GET("/rulesmapping/clusterroles").
 		To(iam.ClusterRulesMapping).
-		Doc("Get cluster role policy rules mapping").
+		Doc("Get the mapping relationships between cluster roles and policy rules.").
+		Returns(http.StatusOK, ok, policy.ClusterRoleRuleMapping).
 		Metadata(restfulspec.KeyOpenAPITags, tags))
 	ws.Route(ws.GET("/rulesmapping/roles").
 		To(iam.RulesMapping).
-		Doc("Get role policy rules mapping").
+		Doc("Get the mapping relationships between namespaced roles and policy rules.").
+		Returns(http.StatusOK, ok, policy.RoleRuleMapping).
 		Metadata(restfulspec.KeyOpenAPITags, tags))
-
 	ws.Route(ws.GET("/workspaces/{workspace}/roles").
 		To(iam.ListWorkspaceRoles).
+		Doc("List all workspace roles.").
 		Param(ws.PathParameter("workspace", "workspace name")).
-		Doc("List workspace role").
+		Returns(http.StatusOK, ok, struct {
+			Items      []rbacv1.ClusterRole `json:"items"`
+			TotalCount int                  `json:"total_count"`
+		}{}).
 		Metadata(restfulspec.KeyOpenAPITags, tags))
 	ws.Route(ws.GET("/workspaces/{workspace}/roles/{role}").
 		To(iam.DescribeWorkspaceRole).
+		Doc("Describes the workspace role.").
 		Param(ws.PathParameter("workspace", "workspace name")).
 		Param(ws.PathParameter("role", "workspace role name")).
-		Doc("Describe workspace role").
+		Returns(http.StatusOK, ok, rbacv1.ClusterRole{}).
 		Metadata(restfulspec.KeyOpenAPITags, tags))
 	ws.Route(ws.GET("/workspaces/{workspace}/roles/{role}/rules").
 		To(iam.ListWorkspaceRoleRules).
+		Doc("List all policy rules of the specified workspace role.").
 		Param(ws.PathParameter("workspace", "workspace name")).
 		Param(ws.PathParameter("role", "workspace role name")).
-		Doc("Get workspace role policy rules").
+		Returns(http.StatusOK, ok, []models.SimpleRule{}).
 		Metadata(restfulspec.KeyOpenAPITags, tags))
 	ws.Route(ws.GET("/workspaces/{workspace}/members").
 		To(iam.ListWorkspaceUsers).
+		Doc("List all members in the specified workspace.").
 		Param(ws.PathParameter("workspace", "workspace name")).
-		Doc("Get workspace member list").
+		Returns(http.StatusOK, ok, pageableUserList).
 		Metadata(restfulspec.KeyOpenAPITags, tags))
 	ws.Route(ws.POST("/workspaces/{workspace}/members").
 		To(iam.InviteUser).
+		Doc("Invite members to a workspace.").
 		Param(ws.PathParameter("workspace", "workspace name")).
-		Doc("Add user to workspace").
+		Reads(models.User{}).
+		Returns(http.StatusOK, ok, errors.Error{}).
 		Metadata(restfulspec.KeyOpenAPITags, tags))
 	ws.Route(ws.DELETE("/workspaces/{workspace}/members/{username}").
 		To(iam.RemoveUser).
+		Doc("Remove members from workspace.").
 		Param(ws.PathParameter("workspace", "workspace name")).
 		Param(ws.PathParameter("name", "username")).
-		Doc("Remove user from workspace").
+		Returns(http.StatusOK, ok, errors.Error{}).
 		Metadata(restfulspec.KeyOpenAPITags, tags))
 	ws.Route(ws.GET("/workspaces/{workspace}/members/{username}").
 		To(iam.DescribeWorkspaceUser).
+		Doc("Describes the specified user.").
 		Param(ws.PathParameter("workspace", "workspace name")).
 		Param(ws.PathParameter("username", "username")).
-		Doc("Describe user in workspace").
-		Metadata(restfulspec.KeyOpenAPITags, tags))
-	ws.Route(ws.GET("/namespaces/{namespace}/roles/{role}/rules").
-		To(iam.ListRoleRules).
-		Param(ws.PathParameter("namespace", "namespace")).
-		Param(ws.PathParameter("role", "role name")).
-		Doc("Get namespace role policy rules").
-		Metadata(restfulspec.KeyOpenAPITags, tags))
-	ws.Route(ws.GET("/devops/{devops}/roles/{role}/rules").
-		To(iam.ListDevopsRoleRules).
-		Param(ws.PathParameter("devops", "devops project id")).
-		Param(ws.PathParameter("role", "devops role name")).
-		Doc("Get devops role policy rules").
+		Returns(http.StatusOK, ok, models.User{}).
 		Metadata(restfulspec.KeyOpenAPITags, tags))
 	c.Add(ws)
 	return nil

pkg/apiserver/iam/am.go

@@ -30,7 +30,7 @@ import (
 	"kubesphere.io/kubesphere/pkg/models/iam/policy"
 )
 
-type roleList struct {
+type RoleList struct {
 	ClusterRoles []*v1.ClusterRole `json:"clusterRole" protobuf:"bytes,2,rep,name=clusterRoles"`
 	Roles        []*v1.Role        `json:"roles" protobuf:"bytes,2,rep,name=roles"`
 }
@@ -132,7 +132,7 @@ func ListUserRoles(req *restful.Request, resp *restful.Response) {
 		return
 	}
 
-	roleList := roleList{}
+	roleList := RoleList{}
 	roleList.Roles = roles
 	roleList.ClusterRoles = clusterRoles
 

pkg/controller/namespace/namespace_controller.go

@@ -35,6 +35,8 @@ import (
 	"kubesphere.io/kubesphere/pkg/constants"
 	"kubesphere.io/kubesphere/pkg/simple/client/openpitrix"
 	"kubesphere.io/kubesphere/pkg/utils/k8sutil"
+	"kubesphere.io/kubesphere/pkg/utils/sliceutil"
+	"math"
 	"reflect"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/controller"
@@ -44,6 +46,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
 	logf "sigs.k8s.io/controller-runtime/pkg/runtime/log"
 	"sigs.k8s.io/controller-runtime/pkg/source"
+	"time"
 )
 
 const (
@@ -115,25 +118,47 @@ func (r *ReconcileNamespace) Reconcile(request reconcile.Request) (reconcile.Res
 		if errors.IsNotFound(err) {
 			// Object not found, return.  Created objects are automatically garbage collected.
 			// For additional cleanup logic use finalizers.
+			// The object is being deleted
+			// our finalizer is present, so lets handle our external dependency
 			return reconcile.Result{}, nil
 		}
 		// Error reading the object - requeue the request.
 		return reconcile.Result{}, err
 	}
 
-	if !instance.ObjectMeta.DeletionTimestamp.IsZero() {
-		// The object is being deleted
+	// name of your custom finalizer
+	finalizer := "finalizers.kubesphere.io/namespaces"
 
+	if instance.ObjectMeta.DeletionTimestamp.IsZero() {
+		// The object is not being deleted, so if it does not have our finalizer,
+		// then lets add the finalizer and update the object.
+		if !sliceutil.HasString(instance.ObjectMeta.Finalizers, finalizer) {
+			instance.ObjectMeta.Finalizers = append(instance.ObjectMeta.Finalizers, finalizer)
+			if err := r.Update(context.Background(), instance); err != nil {
+				return reconcile.Result{}, err
+			}
+		}
+	} else {
+		// The object is being deleted
+		if sliceutil.HasString(instance.ObjectMeta.Finalizers, finalizer) {
 			if err := r.deleteRouter(instance.Name); err != nil {
 				return reconcile.Result{}, err
 			}
 
-		if err := r.deleteRuntime(instance); err != nil {
-			// if fail to delete the external dependency here, return with error
-			// so that it can be retried
+			// delete runtime in the background, retry 3 times
+			go r.deleteRuntime(instance)
+
+			// remove our finalizer from the list and update it.
+			instance.ObjectMeta.Finalizers = sliceutil.RemoveString(instance.ObjectMeta.Finalizers, func(item string) bool {
+				return item == finalizer
+			})
+
+			if err := r.Update(context.Background(), instance); err != nil {
 				return reconcile.Result{}, err
 			}
+		}
 
+		// Our finalizer has finished, so the reconciler can do nothing.
 		return reconcile.Result{}, nil
 	}
 
@@ -347,9 +372,17 @@ func (r *ReconcileNamespace) checkAndCreateRuntime(namespace *corev1.Namespace)
 func (r *ReconcileNamespace) deleteRuntime(namespace *corev1.Namespace) error {
 
 	if runtimeId := namespace.Annotations[constants.OpenPitrixRuntimeAnnotationKey]; runtimeId != "" {
+		maxRetries := float64(3)
+		for i := float64(0); i < maxRetries; i++ {
+			time.Sleep(time.Duration(i*math.Pow(2, i)) * time.Second)
 			log.Info("Deleting openpitrix runtime", "namespace", namespace.Name, "runtime", runtimeId)
-		if err := openpitrix.Client().DeleteRuntime(runtimeId); err != nil {
-			return err
+			err := openpitrix.Client().DeleteRuntime(runtimeId)
+
+			if err == nil || openpitrix.IsNotFound(err) || openpitrix.IsDeleted(err) {
+				return nil
+			}
+
+			log.Error(err, fmt.Sprintf("Deleteing openpitrix runtime failed: %v times left", maxRetries-i-1))
 		}
 	}
 

pkg/models/iam/im.go

@@ -284,7 +284,7 @@ func ListUsers(conditions *params.Conditions, orderBy string, reverse bool, limi
 
 	defer conn.Close()
 
-	pageControl := ldap.NewControlPaging(80)
+	pageControl := ldap.NewControlPaging(1000)
 
 	users := make([]models.User, 0)
 

pkg/models/iam/policy/policy.go

@@ -905,7 +905,7 @@ var (
 					Rules: []v1.PolicyRule{
 						{
 							Verbs:     []string{"get", "list"},
-							APIGroups: []string{"resources.kubesphere.io"},
+							APIGroups: []string{"resources.kubesphere.io", "app.k8s.io"},
 							Resources: []string{"applications"},
 						}, {
 							Verbs:     []string{"get", "list"},
@@ -927,7 +927,7 @@ var (
 					Rules: []v1.PolicyRule{
 						{
 							Verbs:     []string{"create", "update", "patch"},
-							APIGroups: []string{"resources.kubesphere.io"},
+							APIGroups: []string{"resources.kubesphere.io", "app.k8s.io"},
 							Resources: []string{"applications"},
 						}, {
 							Verbs:     []string{"create", "update", "patch"},
@@ -940,7 +940,7 @@ var (
 					Rules: []v1.PolicyRule{
 						{
 							Verbs:     []string{"delete"},
-							APIGroups: []string{"resources.kubesphere.io"},
+							APIGroups: []string{"resources.kubesphere.io", "app.k8s.io"},
 							Resources: []string{"applications"},
 						},
 						{

pkg/simple/client/openpitrix/openpitrixclient.go

@@ -24,7 +24,9 @@ import (
 	"fmt"
 	"github.com/golang/glog"
 	"io/ioutil"
+	"kubesphere.io/kubesphere/pkg/utils/sliceutil"
 	"net/http"
+	"strings"
 	"sync"
 )
 
@@ -48,6 +50,10 @@ type Interface interface {
 	CreateRuntime(runtime *RunTime) error
 	DeleteRuntime(runtimeId string) error
 }
+type cluster struct {
+	Status    string `json:"status"`
+	ClusterId string `json:"cluster_id"`
+}
 
 type Error struct {
 	status  int
@@ -78,14 +84,12 @@ func (c client) CreateRuntime(runtime *RunTime) error {
 
 	data, err := json.Marshal(runtime)
 	if err != nil {
-		glog.Error(err)
 		return err
 	}
 
 	req, err := http.NewRequest(http.MethodPost, fmt.Sprintf("%s/v1/runtimes", openpitrixAPIServer), bytes.NewReader(data))
 
 	if err != nil {
-		glog.Error(err)
 		return err
 	}
 	req.Header.Add("Content-Type", "application/json")
@@ -106,30 +110,46 @@ func (c client) CreateRuntime(runtime *RunTime) error {
 	}
 
 	if resp.StatusCode > http.StatusOK {
-		return Error{resp.StatusCode, string(data)}
+		err = Error{resp.StatusCode, string(data)}
+		glog.Error(err)
+		return err
 	}
 
 	return nil
 }
 
-func (c client) DeleteRuntime(runtimeId string) error {
-	data := []byte(fmt.Sprintf(`{"runtime_id":"%s"}`, runtimeId))
-	req, err := http.NewRequest(http.MethodDelete, fmt.Sprintf("%s/v1/runtimes", openpitrixAPIServer), bytes.NewReader(data))
+func (c client) deleteClusters(clusters []cluster) error {
+	clusterId := make([]string, 0)
 
-	if err != nil {
-		glog.Error(err)
+	for _, cluster := range clusters {
+		if cluster.Status != "deleted" && cluster.Status != "deleting" && !sliceutil.HasString(clusterId, cluster.ClusterId) {
+			clusterId = append(clusterId, cluster.ClusterId)
+		}
+	}
 
-		return err
+	if len(clusterId) == 0 {
+		return nil
+	}
+
+	deleteRequest := struct {
+		ClusterId []string `json:"cluster_id"`
+	}{
+		ClusterId: clusterId,
 	}
+	data, _ := json.Marshal(deleteRequest)
+	req, err := http.NewRequest(http.MethodPost, fmt.Sprintf("%s/v1/clusters/delete", openpitrixAPIServer), bytes.NewReader(data))
 
+	if err != nil {
+		return err
+	}
 	req.Header.Add("Authorization", openpitrixProxyToken)
 
 	resp, err := c.client.Do(req)
-
 	if err != nil {
 		glog.Error(err)
 		return err
 	}
+
 	defer resp.Body.Close()
 	data, err = ioutil.ReadAll(resp.Body)
 
@@ -139,8 +159,110 @@ func (c client) DeleteRuntime(runtimeId string) error {
 	}
 
 	if resp.StatusCode > http.StatusOK {
-		return Error{resp.StatusCode, string(data)}
+		err = Error{resp.StatusCode, string(data)}
+		glog.Error(err)
+		return err
 	}
 
 	return nil
 }
+
+func (c client) listClusters(runtimeId string) ([]cluster, error) {
+	limit := 200
+	offset := 0
+	clusters := make([]cluster, 0)
+	for {
+		req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("%s/v1/clusters?runtime_id=%s&limit=%d&offset=%d", openpitrixAPIServer, runtimeId, limit, offset), nil)
+
+		if err != nil {
+			glog.Error(err)
+			return nil, err
+		}
+
+		req.Header.Add("Authorization", openpitrixProxyToken)
+
+		resp, err := c.client.Do(req)
+
+		if err != nil {
+			glog.Error(err)
+			return nil, err
+		}
+
+		data, err := ioutil.ReadAll(resp.Body)
+
+		if err != nil {
+			glog.Error(err)
+			return nil, err
+		}
+
+		resp.Body.Close()
+
+		if resp.StatusCode > http.StatusOK {
+			err = Error{resp.StatusCode, string(data)}
+			glog.Error(err)
+			return nil, err
+		}
+		listClusterResponse := struct {
+			TotalCount int       `json:"total_count"`
+			ClusterSet []cluster `json:"cluster_set"`
+		}{}
+		err = json.Unmarshal(data, &listClusterResponse)
+
+		if err != nil {
+			glog.Error(err)
+			return nil, err
+		}
+
+		clusters = append(clusters, listClusterResponse.ClusterSet...)
+
+		if listClusterResponse.TotalCount <= limit+offset {
+			break
+		}
+
+		offset += limit
+	}
+
+	return clusters, nil
+}
+
+func (c client) DeleteRuntime(runtimeId string) error {
+	clusters, err := c.listClusters(runtimeId)
+
+	if err != nil {
+		glog.Error(err)
+		return err
+	}
+
+	err = c.deleteClusters(clusters)
+
+	if err != nil {
+		glog.Error(err)
+		return err
+	}
+
+	return nil
+}
+
+func IsNotFound(err error) bool {
+	if e, ok := err.(Error); ok {
+		if e.status == http.StatusNotFound {
+			return true
+		}
+		if strings.Contains(e.message, "not exist") {
+			return true
+		}
+		if strings.Contains(e.message, "not found") {
+			return true
+		}
+	}
+	return false
+}
+
+func IsDeleted(err error) bool {
+	if e, ok := err.(Error); ok {
+		if strings.Contains(e.message, "is [deleted]") {
+			return true
+		}
+	}
+	return false
+}