提交 dd963c0b 编写于 作者: H hongming 提交者: zryfish

fix: ks-account abnormal restart

Signed-off-by: Nhongming <talonwan@yunify.com>
上级 5c8a087a
......@@ -67,6 +67,8 @@ func Run(s *options.ServerRunOptions) error {
return err
}
waitForResourceSync()
err = iam.Init(s.AdminEmail, s.AdminPassword, expireTime)
jwtutil.Setup(s.JWTSecret)
......@@ -74,8 +76,6 @@ func Run(s *options.ServerRunOptions) error {
return err
}
waitForResourceSync()
container := runtime.Container
container.Filter(filter.Logging)
......
......@@ -72,7 +72,6 @@ func (h Auth) ServeHTTP(resp http.ResponseWriter, req *http.Request) (int, error
token, err := h.Validate(uToken)
if err != nil {
log.Println(uToken)
return h.HandleUnauthorized(resp, err), nil
}
......
......@@ -24,6 +24,7 @@ import (
"k8s.io/apiserver/pkg/authorization/authorizer"
"k8s.io/apiserver/pkg/endpoints/request"
"kubesphere.io/kubesphere/pkg/utils/k8sutil"
"log"
"net/http"
"strings"
......@@ -82,6 +83,7 @@ func (c Authentication) ServeHTTP(w http.ResponseWriter, r *http.Request) (int,
func handleForbidden(w http.ResponseWriter, err error) int {
message := fmt.Sprintf("Forbidden,%s", err.Error())
w.Header().Add("WWW-Authenticate", message)
log.Println(message)
return http.StatusForbidden
}
......@@ -94,6 +96,7 @@ func permissionValidate(attrs authorizer.Attributes) (bool, error) {
permitted, err := clusterRoleValidate(attrs)
if err != nil {
log.Println("lister error", err)
return false, err
}
......@@ -105,6 +108,7 @@ func permissionValidate(attrs authorizer.Attributes) (bool, error) {
permitted, err = roleValidate(attrs)
if err != nil {
log.Println("lister error", err)
return false, err
}
......
......@@ -20,6 +20,7 @@ package iam
import (
"github.com/dgrijalva/jwt-go"
"github.com/emicklei/go-restful"
"github.com/golang/glog"
"kubesphere.io/kubesphere/pkg/utils/iputil"
"kubesphere.io/kubesphere/pkg/utils/jwtutil"
"net/http"
......@@ -97,6 +98,7 @@ func TokenReviewHandler(req *restful.Request, resp *restful.Response) {
token, err := jwtutil.ValidateToken(uToken)
if err != nil {
glog.Errorln("token review failed", uToken, err)
failed := TokenReview{APIVersion: APIVersion,
Kind: KindTokenReview,
Status: &Status{
......
......@@ -26,7 +26,7 @@ import (
"github.com/emicklei/go-restful"
"github.com/go-ldap/ldap"
rbacv1 "k8s.io/api/rbac/v1"
"kubesphere.io/kubesphere/pkg/constants"
"kubesphere.io/kubesphere/pkg/errors"
"kubesphere.io/kubesphere/pkg/models"
......@@ -126,7 +126,14 @@ func UpdateUser(req *restful.Request, resp *restful.Response) {
// change password by self
if usernameInHeader == user.Username && user.Password != "" {
_, err = iam.Login(usernameInHeader, user.CurrentPassword, "")
isUserManager, err := isUserManager(usernameInHeader)
if err != nil {
resp.WriteHeaderAndEntity(http.StatusBadRequest, errors.Wrap(err))
return
}
if !isUserManager {
_, err = iam.Login(usernameInHeader, user.CurrentPassword, "")
}
if err != nil {
resp.WriteHeaderAndEntity(http.StatusBadRequest, errors.Wrap(fmt.Errorf("incorrect current password")))
return
......@@ -143,6 +150,17 @@ func UpdateUser(req *restful.Request, resp *restful.Response) {
resp.WriteAsJson(result)
}
func isUserManager(username string) (bool, error) {
rules, err := iam.GetUserClusterRules(username)
if err != nil {
return false, err
}
if iam.RulesMatchesRequired(rules, rbacv1.PolicyRule{Verbs: []string{"update"}, Resources: []string{"users"}, APIGroups: []string{"iam.kubesphere.io"}}) {
return true, nil
}
return false, nil
}
func UserLoginLog(req *restful.Request, resp *restful.Response) {
username := req.PathParameter("name")
logs, err := iam.LoginLog(username)
......
......@@ -18,7 +18,6 @@
package tenant
import (
"fmt"
"github.com/emicklei/go-restful"
"github.com/golang/glog"
"k8s.io/api/core/v1"
......@@ -31,11 +30,14 @@ import (
"kubesphere.io/kubesphere/pkg/errors"
"kubesphere.io/kubesphere/pkg/models"
"kubesphere.io/kubesphere/pkg/models/iam"
"kubesphere.io/kubesphere/pkg/models/metrics"
"kubesphere.io/kubesphere/pkg/models/resources"
"kubesphere.io/kubesphere/pkg/models/tenant"
"kubesphere.io/kubesphere/pkg/models/workspaces"
"kubesphere.io/kubesphere/pkg/params"
"kubesphere.io/kubesphere/pkg/simple/client/elasticsearch"
"kubesphere.io/kubesphere/pkg/simple/client/kubesphere"
"kubesphere.io/kubesphere/pkg/utils/sliceutil"
"net/http"
"strings"
)
......@@ -123,6 +125,22 @@ func ListNamespaces(req *restful.Request, resp *restful.Response) {
return
}
namespaces := make([]*v1.Namespace, 0)
for _, item := range result.Items {
namespaces = append(namespaces, item.(*v1.Namespace).DeepCopy())
}
namespaces = metrics.GetNamespacesWithMetrics(namespaces)
items := make([]interface{}, 0)
for _, item := range namespaces {
items = append(items, item)
}
result.Items = items
resp.WriteAsJson(result)
}
......@@ -298,37 +316,84 @@ func LogQuery(req *restful.Request, resp *restful.Response) {
username := req.HeaderParameter(constants.UserNameHeader)
mapping, err := iam.GetUserWorkspaceRoleMap(username)
if err != nil {
resp.WriteError(http.StatusInternalServerError, err)
glog.Errorln(err)
return
}
workspaces := make([]string, 0)
for workspaceName, role := range mapping {
if role == fmt.Sprintf("workspace:%s:admin", workspaceName) {
workspaces = append(workspaces, workspaceName)
}
}
// regenerate the request for log query
newUrl := net.FormatURL("http", "127.0.0.1", 80, "/kapis/logging.kubesphere.io/v1alpha2/cluster")
values := req.Request.URL.Query()
rules, err := iam.GetUserClusterRules(username)
clusterRules, err := iam.GetUserClusterRules(username)
if err != nil {
resp.WriteError(http.StatusInternalServerError, err)
glog.Errorln(err)
return
}
if !iam.RulesMatchesRequired(rules, rbacv1.PolicyRule{Verbs: []string{"get"}, Resources: []string{"*"}, APIGroups: []string{"logging.kubesphere.io"}}) {
values.Set("workspaces", strings.Join(workspaces, ","))
hasClusterLogAccess := iam.RulesMatchesRequired(clusterRules, rbacv1.PolicyRule{Verbs: []string{"get"}, Resources: []string{"*"}, APIGroups: []string{"logging.kubesphere.io"}})
// if the user is not a cluster admin
if !hasClusterLogAccess {
queryNamespaces := strings.Split(req.QueryParameter("namespaces"), ",")
// then the user can only view logs of namespaces he belongs to
namespaces := make([]string, 0)
roles, err := iam.GetUserRoles("", username)
if err != nil {
resp.WriteError(http.StatusInternalServerError, err)
glog.Errorln(err)
}
for _, role := range roles {
if !sliceutil.HasString(namespaces, role.Namespace) && iam.RulesMatchesRequired(role.Rules, rbacv1.PolicyRule{Verbs: []string{"get"}, Resources: []string{"*"}, APIGroups: []string{"logging.kubesphere.io"}}) {
namespaces = append(namespaces, role.Namespace)
}
}
// if the user belongs to no namespace
// then no log visible
if len(namespaces) == 0 {
res := esclient.QueryResult{Status: http.StatusOK}
resp.WriteAsJson(res)
return
} else if len(queryNamespaces) == 1 && queryNamespaces[0] == "" {
values.Set("namespaces", strings.Join(namespaces, ","))
} else {
inter := intersection(queryNamespaces, namespaces)
if len(inter) == 0 {
res := esclient.QueryResult{Status: http.StatusOK}
resp.WriteAsJson(res)
return
}
values.Set("namespaces", strings.Join(inter, ","))
}
}
newUrl.RawQuery = values.Encode()
// forward the request to logging model
newHttpRequest, _ := http.NewRequest(http.MethodGet, newUrl.String(), nil)
logging.LoggingQueryCluster(restful.NewRequest(newHttpRequest), resp)
}
func intersection(s1, s2 []string) (inter []string) {
hash := make(map[string]bool)
for _, e := range s1 {
hash[e] = true
}
for _, e := range s2 {
// If elements present in the hashmap then append intersection list.
if hash[e] {
inter = append(inter, e)
}
}
//Remove dups from slice.
inter = removeDups(inter)
return
}
//Remove dups from slice.
func removeDups(elements []string) (nodups []string) {
encountered := make(map[string]bool)
for _, element := range elements {
if !encountered[element] {
nodups = append(nodups, element)
encountered[element] = true
}
}
return
}
......@@ -142,9 +142,11 @@ func (r *ReconcileClusterRoleBinding) updateRoleBindings(clusterRoleBinding *rba
log.Info("Creating default role binding", "namespace", namespace.Name, "name", adminBinding.Name)
err = r.Create(context.TODO(), adminBinding)
if err != nil {
return err
log.Error(err, "default role binding create failed", "namespace", namespace.Name, "name", adminBinding.Name)
}
return err
} else if err != nil {
log.Error(err, "default role binding not found", "namespace", namespace.Name, "name", adminBinding.Name)
return err
}
......@@ -182,9 +184,7 @@ func (r *ReconcileClusterRoleBinding) updateRoleBindings(clusterRoleBinding *rba
if errors.IsNotFound(err) {
log.Info("Creating default role binding", "namespace", namespace.Name, "name", viewerBinding.Name)
err = r.Create(context.TODO(), viewerBinding)
if err != nil {
return err
}
return err
} else if err != nil {
return err
}
......
......@@ -232,6 +232,7 @@ func (r *ReconcileNamespace) checkAndCreateRoleBindings(namespace *corev1.Namesp
if err != nil {
return err
}
found = adminBinding
} else if err != nil {
return err
}
......@@ -276,6 +277,7 @@ func (r *ReconcileNamespace) checkAndCreateRoleBindings(namespace *corev1.Namesp
if err != nil {
return err
}
found = viewerBinding
} else if err != nil {
return err
}
......@@ -309,7 +311,8 @@ func (r *ReconcileNamespace) checkAndCreateRuntime(namespace *corev1.Namespace)
}
cm := &corev1.ConfigMap{}
err := r.Get(context.TODO(), types.NamespacedName{Namespace: constants.KubeSphereControlNamespace, Name: constants.AdminUserName}, cm)
configName := fmt.Sprintf("kubeconfig-%s", constants.AdminUserName)
err := r.Get(context.TODO(), types.NamespacedName{Namespace: constants.KubeSphereControlNamespace, Name: configName}, cm)
if err != nil {
return err
......@@ -353,12 +356,8 @@ func (r *ReconcileNamespace) checkAndBindWorkspace(namespace *corev1.Namespace)
if err != nil {
if errors.IsNotFound(err) {
log.Error(err, "namespace", namespace.Name)
delete(namespace.Labels, constants.WorkspaceLabelKey)
err = r.Update(context.TODO(), namespace)
if err != nil {
return err
}
log.Error(err, fmt.Sprintf("namespace %s bind workspace %s but not found", namespace.Name, workspaceName))
return nil
}
return err
}
......
......@@ -188,9 +188,10 @@ func (r *ReconcileWorkspace) createWorkspaceAdmin(instance *tenantv1alpha1.Works
}
// Update the found object and write the result back if there are any changes
if !reflect.DeepEqual(admin.Rules, found.Rules) || !reflect.DeepEqual(admin.Labels, found.Labels) {
if !reflect.DeepEqual(admin.Rules, found.Rules) || !reflect.DeepEqual(admin.Labels, found.Labels) || !reflect.DeepEqual(admin.Annotations, found.Annotations) {
found.Rules = admin.Rules
found.Labels = admin.Labels
found.Annotations = admin.Annotations
log.Info("Updating workspace role", "workspace", instance.Name, "name", admin.Name)
err = r.Update(context.TODO(), found)
if err != nil {
......@@ -226,9 +227,10 @@ func (r *ReconcileWorkspace) createWorkspaceRegular(instance *tenantv1alpha1.Wor
}
// Update the found object and write the result back if there are any changes
if !reflect.DeepEqual(regular.Rules, found.Rules) || !reflect.DeepEqual(regular.Labels, found.Labels) {
if !reflect.DeepEqual(regular.Rules, found.Rules) || !reflect.DeepEqual(regular.Labels, found.Labels) || !reflect.DeepEqual(regular.Annotations, found.Annotations) {
found.Rules = regular.Rules
found.Labels = regular.Labels
found.Annotations = regular.Annotations
log.Info("Updating workspace role", "workspace", instance.Name, "name", regular.Name)
err = r.Update(context.TODO(), found)
if err != nil {
......@@ -264,9 +266,10 @@ func (r *ReconcileWorkspace) createWorkspaceViewer(instance *tenantv1alpha1.Work
}
// Update the found object and write the result back if there are any changes
if !reflect.DeepEqual(viewer.Rules, found.Rules) || !reflect.DeepEqual(viewer.Labels, found.Labels) {
if !reflect.DeepEqual(viewer.Rules, found.Rules) || !reflect.DeepEqual(viewer.Labels, found.Labels) || !reflect.DeepEqual(viewer.Annotations, found.Annotations) {
found.Rules = viewer.Rules
found.Labels = viewer.Labels
found.Annotations = viewer.Annotations
log.Info("Updating workspace role", "workspace", instance.Name, "name", viewer.Name)
err = r.Update(context.TODO(), found)
if err != nil {
......
此差异已折叠。
......@@ -18,8 +18,10 @@
package iam
import (
"encoding/json"
"errors"
"fmt"
"io/ioutil"
"kubesphere.io/kubesphere/pkg/constants"
"kubesphere.io/kubesphere/pkg/informers"
"kubesphere.io/kubesphere/pkg/params"
......@@ -51,6 +53,10 @@ var (
tokenExpireTime time.Duration
)
const (
userInitFile = "/etc/ks-iam/users.json"
)
func Init(email, password string, t time.Duration) error {
adminEmail = email
adminPassword = password
......@@ -67,12 +73,18 @@ func Init(email, password string, t time.Duration) error {
err = checkAndCreateDefaultUser(conn)
if err != nil {
glog.Errorln("create default users", err)
return err
}
err = checkAndCreateDefaultGroup(conn)
return err
if err != nil {
glog.Errorln("create default groups", err)
return err
}
return nil
}
func checkAndCreateDefaultGroup(conn ldap.Client) error {
......@@ -124,10 +136,19 @@ func checkAndCreateDefaultUser(conn ldap.Client) error {
return fmt.Errorf("iam database init failed: %s\n", err)
}
if users == nil || len(users.Entries) == 0 {
_, err := CreateUser(&models.User{Username: constants.AdminUserName, Email: adminEmail, Password: adminPassword, Description: "Administrator account that was always created by default."})
if err != nil {
return fmt.Errorf("admin create failed: %s\n", err)
data, err := ioutil.ReadFile(userInitFile)
var initUsers []models.User
if err == nil {
json.Unmarshal(data, &initUsers)
}
initUsers = append(initUsers, models.User{Username: constants.AdminUserName, Email: adminEmail, Password: adminPassword, Description: "Administrator account that was always created by default.", ClusterRole: constants.ClusterAdmin})
if users == nil || len(users.Entries) < len(initUsers) {
for _, user := range initUsers {
_, err = CreateUser(&user)
if err != nil && !ldap.IsErrorWithCode(err, ldap.LDAPResultEntryAlreadyExists) {
return fmt.Errorf("user %s init failed: %s\n", user.Username, err)
}
}
}
......@@ -192,6 +213,7 @@ func Login(username string, password string, ip string) (*models.Token, error) {
err = conn.Bind(dn, password)
if err != nil {
glog.Errorln("auth error", username, err)
return nil, err
}
......@@ -228,55 +250,6 @@ func LoginLog(username string) ([]string, error) {
return data, nil
}
func ListUsersByName(names []string) (*models.PageableResponse, error) {
users := make([]*models.User, 0)
for _, name := range names {
if !k8sutil.ContainsUser(users, name) {
user, err := GetUserInfo(name)
if err != nil {
if ldap.IsErrorWithCode(err, ldap.LDAPResultNoSuchObject) {
continue
}
return nil, err
}
users = append(users, user)
}
}
items := make([]interface{}, 0)
for _, u := range users {
items = append(items, u)
}
return &models.PageableResponse{Items: items, TotalCount: len(items)}, nil
}
func ListUserByEmail(email []string) (*models.PageableResponse, error) {
users := make([]*models.User, 0)
for _, mail := range email {
user, err := GetUserInfoByEmail(mail)
if err != nil {
if ldap.IsErrorWithCode(err, ldap.LDAPResultNoSuchObject) {
continue
}
return nil, err
}
if !k8sutil.ContainsUser(users, user.Username) {
users = append(users, user)
}
}
items := make([]interface{}, 0)
for _, u := range users {
items = append(items, u)
}
return &models.PageableResponse{Items: items, TotalCount: len(items)}, nil
}
func ListUsers(conditions *params.Conditions, orderBy string, reverse bool, limit, offset int) (*models.PageableResponse, error) {
conn, err := ldapclient.Client()
......@@ -325,6 +298,7 @@ func ListUsers(conditions *params.Conditions, orderBy string, reverse bool, limi
response, err := conn.Search(userSearchRequest)
if err != nil {
glog.Errorln("search user", err)
return nil, err
}
......@@ -396,55 +370,15 @@ func DescribeUser(username string) (*models.User, error) {
groups, err := GetUserGroups(username)
if err != nil {
return nil, err
if err == nil {
user.Groups = groups
}
user.Groups = groups
user.AvatarUrl = getAvatar(username)
return user, nil
}
func GetUserInfoByEmail(mail string) (*models.User, error) {
conn, err := ldapclient.Client()
if err != nil {
return nil, err
}
userSearchRequest := ldap.NewSearchRequest(
ldapclient.UserSearchBase,
ldap.ScopeWholeSubtree, ldap.NeverDerefAliases, 0, 0, false,
fmt.Sprintf("(&(objectClass=inetOrgPerson)(mail=%s))", mail),
[]string{"uid", "description", "preferredLanguage", "createTimestamp"},
nil,
)
result, err := conn.Search(userSearchRequest)
if err != nil {
return nil, err
}
if len(result.Entries) != 1 {
return nil, ldap.NewError(ldap.LDAPResultNoSuchObject, fmt.Errorf("user %s does not exist", mail))
}
username := result.Entries[0].GetAttributeValue("uid")
description := result.Entries[0].GetAttributeValue("description")
lang := result.Entries[0].GetAttributeValue("preferredLanguage")
createTimestamp, _ := time.Parse("20060102150405Z", result.Entries[0].GetAttributeValue("createTimestamp"))
user := &models.User{Username: username, Email: mail, Description: description, Lang: lang, CreateTime: createTimestamp}
user.LastLoginTime = getLastLoginTime(username)
clusterRole, err := GetUserClusterRole(user.Username)
if err != nil {
return nil, err
}
user.ClusterRole = clusterRole.Name
return user, nil
}
// Get user info only included email description & lang
func GetUserInfo(username string) (*models.User, error) {
......@@ -465,6 +399,7 @@ func GetUserInfo(username string) (*models.User, error) {
result, err := conn.Search(userSearchRequest)
if err != nil {
glog.Errorln("search user", err)
return nil, err
}
......@@ -567,6 +502,7 @@ func DeleteUser(username string) error {
err = conn.Del(deleteRequest)
if err != nil {
glog.Errorln("delete user", err)
return err
}
......@@ -673,14 +609,11 @@ func UserCreateCheck(check string) (exist bool, err error) {
result, err := conn.Search(userSearchRequest)
if err != nil {
glog.Errorln("search user", err)
return false, err
}
if len(result.Entries) > 0 {
return true, nil
} else {
return false, nil
}
return len(result.Entries) > 0, nil
}
func CreateUser(user *models.User) (*models.User, error) {
......@@ -708,6 +641,7 @@ func CreateUser(user *models.User) (*models.User, error) {
result, err := conn.Search(userSearchRequest)
if err != nil {
glog.Errorln("search user", err)
return nil, err
}
......@@ -718,6 +652,7 @@ func CreateUser(user *models.User) (*models.User, error) {
maxUid, err := getMaxUid(conn)
if err != nil {
glog.Errorln("get max uid", err)
return nil, err
}
......@@ -743,6 +678,7 @@ func CreateUser(user *models.User) (*models.User, error) {
err = conn.Add(userCreateRequest)
if err != nil {
glog.Errorln("create user", err)
return nil, err
}
......@@ -754,6 +690,7 @@ func CreateUser(user *models.User) (*models.User, error) {
err := CreateClusterRoleBinding(user.Username, user.ClusterRole)
if err != nil {
glog.Errorln("create cluster role binding filed", err)
return nil, err
}
}
......@@ -864,6 +801,7 @@ func UpdateUser(user *models.User) (*models.User, error) {
err = CreateClusterRoleBinding(user.Username, user.ClusterRole)
if err != nil {
glog.Errorln("create cluster role binding filed", err)
return nil, err
}
......@@ -884,6 +822,7 @@ func DeleteGroup(path string) error {
err = conn.Del(groupDeleteRequest)
if err != nil {
glog.Errorln("delete user group", err)
return err
}
......@@ -903,6 +842,7 @@ func CreateGroup(group *models.Group) (*models.Group, error) {
maxGid, err := getMaxGid(conn)
if err != nil {
glog.Errorln("get max gid", err)
return nil, err
}
......@@ -930,6 +870,7 @@ func CreateGroup(group *models.Group) (*models.Group, error) {
err = conn.Add(groupCreateRequest)
if err != nil {
glog.Errorln("create group", err)
return nil, err
}
......@@ -976,45 +917,13 @@ func UpdateGroup(group *models.Group) (*models.Group, error) {
err = conn.Modify(groupUpdateRequest)
if err != nil {
glog.Errorln("update group", err)
return nil, err
}
return group, nil
}
func CountChild(path string) (int, error) {
// bind root DN
conn, err := ldapclient.Client()
if err != nil {
return 0, err
}
defer conn.Close()
var groupSearchRequest *ldap.SearchRequest
if path == "" {
groupSearchRequest = ldap.NewSearchRequest(ldapclient.GroupSearchBase,
ldap.ScopeSingleLevel, ldap.NeverDerefAliases, 0, 0, false,
"(&(objectClass=posixGroup))",
[]string{"cn", "gidNumber", "memberUid", "description"},
nil)
} else {
searchBase, cn := splitPath(path)
groupSearchRequest = ldap.NewSearchRequest(fmt.Sprintf("cn=%s,%s", cn, searchBase),
ldap.ScopeSingleLevel, ldap.NeverDerefAliases, 0, 0, false,
"(&(objectClass=posixGroup))",
[]string{"cn", "gidNumber", "memberUid", "description"},
nil)
}
result, err := conn.Search(groupSearchRequest)
if err != nil {
return 0, err
}
return len(result.Entries), nil
}
func ChildList(path string) ([]models.Group, error) {
// bind root DN
......@@ -1105,6 +1014,7 @@ func DescribeGroup(path string) (*models.Group, error) {
result, err := conn.Search(groupSearchRequest)
if err != nil {
glog.Errorln("search group", err)
return nil, err
}
......
......@@ -101,11 +101,6 @@ var (
APIGroups: []string{"rbac.authorization.k8s.io"},
Resources: []string{"rolebindings", "roles"},
},
{
Verbs: []string{"*"},
APIGroups: []string{"jenkins.kubesphere.io", "devops.kubesphere.io"},
Resources: []string{"*"},
},
},
},
},
......
......@@ -26,6 +26,7 @@ import (
"crypto/x509/pkix"
"encoding/base64"
"encoding/pem"
"fmt"
"io/ioutil"
"kubesphere.io/kubesphere/pkg/simple/client/k8s"
"math/big"
......@@ -208,7 +209,7 @@ func generateCaAndKey(user, caPath, keyPath string) (string, string, error) {
return base64Cert, base64Key, nil
}
func createKubeConfig(userName string) (string, error) {
func createKubeConfig(username string) (string, error) {
tmpKubeConfig := kubeConfig{ApiVersion: "v1", Kind: "Config"}
serverCa, err := ioutil.ReadFile(caPath)
if err != nil {
......@@ -220,17 +221,17 @@ func createKubeConfig(userName string) (string, error) {
tmpCluster := cluster{Cluster: tmpClusterInfo, Name: clusterName}
tmpKubeConfig.Clusters = append(tmpKubeConfig.Clusters, tmpCluster)
contextName := userName + "@" + clusterName
tmpContext := contextObject{Context: contextInfo{User: userName, Cluster: clusterName, NameSpace: defaultNamespace}, Name: contextName}
contextName := username + "@" + clusterName
tmpContext := contextObject{Context: contextInfo{User: username, Cluster: clusterName, NameSpace: defaultNamespace}, Name: contextName}
tmpKubeConfig.Contexts = append(tmpKubeConfig.Contexts, tmpContext)
cert, key, err := generateCaAndKey(userName, caPath, keyPath)
cert, key, err := generateCaAndKey(username, caPath, keyPath)
if err != nil {
return "", err
}
tmpUser := user{User: userInfo{CaData: cert, KeyData: key}, Name: userName}
tmpUser := user{User: userInfo{CaData: cert, KeyData: key}, Name: username}
tmpKubeConfig.Users = append(tmpKubeConfig.Users, tmpUser)
tmpKubeConfig.CurrentContext = contextName
......@@ -242,23 +243,23 @@ func createKubeConfig(userName string) (string, error) {
return string(config), nil
}
func CreateKubeConfig(user string) error {
func CreateKubeConfig(username string) error {
k8sClient := k8s.Client()
_, err := k8sClient.CoreV1().ConfigMaps(constants.KubeSphereControlNamespace).Get(user, metaV1.GetOptions{})
configName := fmt.Sprintf("kubeconfig-%s", username)
_, err := k8sClient.CoreV1().ConfigMaps(constants.KubeSphereControlNamespace).Get(configName, metaV1.GetOptions{})
if errors.IsNotFound(err) {
config, err := createKubeConfig(user)
config, err := createKubeConfig(username)
if err != nil {
glog.Errorln(err)
return err
}
data := map[string]string{"config": string(config)}
configMap := v1.ConfigMap{TypeMeta: metaV1.TypeMeta{Kind: "Configmap", APIVersion: "v1"}, ObjectMeta: metaV1.ObjectMeta{Name: user}, Data: data}
configMap := v1.ConfigMap{TypeMeta: metaV1.TypeMeta{Kind: "Configmap", APIVersion: "v1"}, ObjectMeta: metaV1.ObjectMeta{Name: configName}, Data: data}
_, err = k8sClient.CoreV1().ConfigMaps(constants.KubeSphereControlNamespace).Create(&configMap)
if err != nil && !errors.IsAlreadyExists(err) {
glog.Errorf("create user %s's kubeConfig failed, reason: %v", user, err)
glog.Errorf("create username %s's kubeConfig failed, reason: %v", username, err)
return err
}
}
......@@ -267,27 +268,29 @@ func CreateKubeConfig(user string) error {
}
func GetKubeConfig(user string) (string, error) {
func GetKubeConfig(username string) (string, error) {
k8sClient := k8s.Client()
configMap, err := k8sClient.CoreV1().ConfigMaps(constants.KubeSphereControlNamespace).Get(user, metaV1.GetOptions{})
configName := fmt.Sprintf("kubeconfig-%s", username)
configMap, err := k8sClient.CoreV1().ConfigMaps(constants.KubeSphereControlNamespace).Get(configName, metaV1.GetOptions{})
if err != nil {
glog.Errorf("cannot get user %s's kubeConfig, reason: %v", user, err)
glog.Errorf("cannot get username %s's kubeConfig, reason: %v", username, err)
return "", err
}
return configMap.Data[kubectlConfigKey], nil
}
func DelKubeConfig(user string) error {
func DelKubeConfig(username string) error {
k8sClient := k8s.Client()
_, err := k8sClient.CoreV1().ConfigMaps(constants.KubeSphereControlNamespace).Get(user, metaV1.GetOptions{})
configName := fmt.Sprintf("kubeconfig-%s", username)
_, err := k8sClient.CoreV1().ConfigMaps(constants.KubeSphereControlNamespace).Get(configName, metaV1.GetOptions{})
if errors.IsNotFound(err) {
return nil
}
deletePolicy := metaV1.DeletePropagationBackground
err = k8sClient.CoreV1().ConfigMaps(constants.KubeSphereControlNamespace).Delete(user, &metaV1.DeleteOptions{PropagationPolicy: &deletePolicy})
err = k8sClient.CoreV1().ConfigMaps(constants.KubeSphereControlNamespace).Delete(configName, &metaV1.DeleteOptions{PropagationPolicy: &deletePolicy})
if err != nil {
glog.Errorf("delete user %s's kubeConfig failed, reason: %v", user, err)
glog.Errorf("delete username %s's kubeConfig failed, reason: %v", username, err)
return err
}
return nil
......
......@@ -23,9 +23,10 @@ import (
"kubesphere.io/kubesphere/pkg/models"
"kubesphere.io/kubesphere/pkg/simple/client/k8s"
"math/rand"
"os"
"github.com/golang/glog"
"k8s.io/api/apps/v1beta2"
appsv1 "k8s.io/api/apps/v1"
"k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/labels"
......@@ -39,9 +40,18 @@ const (
namespace = constants.KubeSphereControlNamespace
)
var DefaultImage = "kubesphere/kubectl:advanced-1.0.0"
func init() {
if env := os.Getenv("KUBECTL_IMAGE"); env != "" {
DefaultImage = env
}
}
func GetKubectlPod(username string) (models.PodInfo, error) {
k8sClient := k8s.Client()
deploy, err := k8sClient.AppsV1beta2().Deployments(namespace).Get(username, metav1.GetOptions{})
deployName := fmt.Sprintf("kubectl-%s", username)
deploy, err := k8sClient.AppsV1().Deployments(namespace).Get(deployName, metav1.GetOptions{})
if err != nil {
glog.Errorln(err)
return models.PodInfo{}, err
......@@ -86,33 +96,35 @@ func selectCorrectPod(namespace string, pods []v1.Pod) (kubectlPod v1.Pod, err e
return kubectlPodList[random], nil
}
func CreateKubectlDeploy(user string) error {
func CreateKubectlDeploy(username string) error {
k8sClient := k8s.Client()
_, err := k8sClient.AppsV1().Deployments(namespace).Get(user, metav1.GetOptions{})
deployName := fmt.Sprintf("kubectl-%s", username)
configName := fmt.Sprintf("kubeconfig-%s", username)
_, err := k8sClient.AppsV1().Deployments(namespace).Get(deployName, metav1.GetOptions{})
if err == nil {
return nil
}
replica := int32(1)
selector := metav1.LabelSelector{MatchLabels: map[string]string{"user": user}}
config := v1.ConfigMapVolumeSource{Items: []v1.KeyToPath{{Key: "config", Path: "config"}}, LocalObjectReference: v1.LocalObjectReference{Name: user}}
deployment := v1beta2.Deployment{
selector := metav1.LabelSelector{MatchLabels: map[string]string{"username": username}}
config := v1.ConfigMapVolumeSource{Items: []v1.KeyToPath{{Key: "config", Path: "config"}}, LocalObjectReference: v1.LocalObjectReference{Name: configName}}
deployment := appsv1.Deployment{
ObjectMeta: metav1.ObjectMeta{
Name: user,
Name: deployName,
},
Spec: v1beta2.DeploymentSpec{
Spec: appsv1.DeploymentSpec{
Replicas: &replica,
Selector: &selector,
Template: v1.PodTemplateSpec{
ObjectMeta: metav1.ObjectMeta{
Labels: map[string]string{
"user": user,
"username": username,
},
},
Spec: v1.PodSpec{
Containers: []v1.Container{
{Name: "kubectl",
Image: "",
Image: DefaultImage,
VolumeMounts: []v1.VolumeMount{{Name: "kubeconfig", MountPath: "/root/.kube"}},
},
},
......@@ -122,28 +134,29 @@ func CreateKubectlDeploy(user string) error {
},
}
_, err = k8sClient.AppsV1beta2().Deployments(namespace).Create(&deployment)
_, err = k8sClient.AppsV1().Deployments(namespace).Create(&deployment)
return err
}
func DelKubectlDeploy(user string) error {
func DelKubectlDeploy(username string) error {
k8sClient := k8s.Client()
_, err := k8sClient.AppsV1beta2().Deployments(namespace).Get(user, metav1.GetOptions{})
deployName := fmt.Sprintf("kubectl-%s", username)
_, err := k8sClient.AppsV1().Deployments(namespace).Get(deployName, metav1.GetOptions{})
if errors.IsNotFound(err) {
return nil
}
if err != nil {
err := fmt.Errorf("delete user %s failed, reason:%v", user, err)
err := fmt.Errorf("delete username %s failed, reason:%v", username, err)
return err
}
deletePolicy := metav1.DeletePropagationBackground
err = k8sClient.AppsV1beta2().Deployments(namespace).Delete(user, &metav1.DeleteOptions{PropagationPolicy: &deletePolicy})
err = k8sClient.AppsV1().Deployments(namespace).Delete(deployName, &metav1.DeleteOptions{PropagationPolicy: &deletePolicy})
if err != nil {
err := fmt.Errorf("delete user %s failed, reason:%v", user, err)
err := fmt.Errorf("delete username %s failed, reason:%v", username, err)
return err
}
......
......@@ -47,7 +47,7 @@ func GetNamespacesWithMetrics(namespaces []*v1.Namespace) []*v1.Namespace {
for _, data := range result.Data.Result {
metricDescMap, ok := data["metric"].(map[string]interface{})
if ok {
if ns, exist := metricDescMap["namespace"]; exist {
if ns, exist := metricDescMap["resource_name"]; exist {
timeAndValue, ok := data["value"].([]interface{})
if ok && len(timeAndValue) == 2 {
for i := 0; i < len(namespaces); i++ {
......
......@@ -25,7 +25,7 @@ import (
"time"
"github.com/golang/glog"
"k8s.io/api/apps/v1beta2"
appsv1 "k8s.io/api/apps/v1"
"k8s.io/api/core/v1"
policy "k8s.io/api/policy/v1beta1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
......@@ -78,7 +78,7 @@ func drainEviction(nodename string, donech chan bool, errch chan error) {
errch <- err
}
options.FieldSelector = ""
daemonsetList, err := k8sclient.AppsV1beta2().DaemonSets("").List(options)
daemonsetList, err := k8sclient.AppsV1().DaemonSets("").List(options)
if err != nil {
......@@ -142,7 +142,7 @@ func isMirrorPod(pod *v1.Pod) bool {
return ok
}
func containDaemonset(pod v1.Pod, daemonsetList v1beta2.DaemonSetList) bool {
func containDaemonset(pod v1.Pod, daemonsetList appsv1.DaemonSetList) bool {
flag := false
for _, daemonset := range daemonsetList.Items {
......
......@@ -41,11 +41,11 @@ func (*clusterRoleSearcher) get(namespace, name string) (interface{}, error) {
func (*clusterRoleSearcher) match(match map[string]string, item *rbac.ClusterRole) bool {
for k, v := range match {
switch k {
case ownerKind:
case OwnerKind:
fallthrough
case ownerName:
kind := match[ownerKind]
name := match[ownerName]
case OwnerName:
kind := match[OwnerKind]
name := match[OwnerName]
if !k8sutil.IsControlledBy(item.OwnerReferences, kind, name) {
return false
}
......@@ -81,7 +81,7 @@ func (*clusterRoleSearcher) fuzzy(fuzzy map[string]string, item *rbac.ClusterRol
if !strings.Contains(item.Name, v) && !strings.Contains(item.Annotations[constants.DisplayNameAnnotationKey], v) {
return false
}
case label:
case Label:
if !searchFuzzy(item.Labels, "", v) {
return false
}
......
......@@ -66,7 +66,7 @@ func (*configMapSearcher) fuzzy(fuzzy map[string]string, item *v1.ConfigMap) boo
if !strings.Contains(item.Name, v) && !strings.Contains(item.Annotations[constants.DisplayNameAnnotationKey], v) {
return false
}
case label:
case Label:
if !searchFuzzy(item.Labels, "", v) {
return false
}
......
......@@ -78,7 +78,7 @@ func (*cronJobSearcher) fuzzy(fuzzy map[string]string, item *v1beta1.CronJob) bo
if !strings.Contains(item.Name, v) && !strings.Contains(item.Annotations[constants.DisplayNameAnnotationKey], v) {
return false
}
case label:
case Label:
if !searchFuzzy(item.Labels, "", v) {
return false
}
......@@ -103,7 +103,7 @@ func (*cronJobSearcher) fuzzy(fuzzy map[string]string, item *v1beta1.CronJob) bo
func (*cronJobSearcher) compare(a, b *v1beta1.CronJob, orderBy string) bool {
switch orderBy {
case lastScheduleTime:
case LastScheduleTime:
if a.Status.LastScheduleTime == nil {
return true
}
......
......@@ -80,7 +80,7 @@ func (*daemonSetSearcher) fuzzy(fuzzy map[string]string, item *v1.DaemonSet) boo
if !strings.Contains(item.Name, v) && !strings.Contains(item.Annotations[constants.DisplayNameAnnotationKey], v) {
return false
}
case label:
case Label:
if !searchFuzzy(item.Labels, "", v) {
return false
}
......
......@@ -84,7 +84,7 @@ func (*deploymentSearcher) fuzzy(fuzzy map[string]string, item *v1.Deployment) b
if !strings.Contains(item.Name, v) && !strings.Contains(item.Annotations[constants.DisplayNameAnnotationKey], v) {
return false
}
case label:
case Label:
if !searchFuzzy(item.Labels, "", v) {
return false
}
......
......@@ -67,7 +67,7 @@ func (*ingressSearcher) fuzzy(fuzzy map[string]string, item *extensions.Ingress)
if !strings.Contains(item.Name, v) && !strings.Contains(item.Annotations[constants.DisplayNameAnnotationKey], v) {
return false
}
case label:
case Label:
if !searchFuzzy(item.Labels, "", v) {
return false
}
......
......@@ -86,7 +86,7 @@ func (*jobSearcher) fuzzy(fuzzy map[string]string, item *batchv1.Job) bool {
if !strings.Contains(item.Name, v) && !strings.Contains(item.Annotations[constants.DisplayNameAnnotationKey], v) {
return false
}
case label:
case Label:
if !searchFuzzy(item.Labels, "", v) {
return false
}
......@@ -126,7 +126,7 @@ func (*jobSearcher) compare(a, b *batchv1.Job, orderBy string) bool {
switch orderBy {
case CreateTime:
return a.CreationTimestamp.Time.Before(b.CreationTimestamp.Time)
case updateTime:
case UpdateTime:
return jobUpdateTime(a).Before(jobUpdateTime(b))
case Name:
fallthrough
......
......@@ -66,7 +66,7 @@ func (*namespaceSearcher) fuzzy(fuzzy map[string]string, item *v1.Namespace) boo
if !strings.Contains(item.Name, v) && !strings.Contains(item.Annotations[constants.DisplayNameAnnotationKey], v) {
return false
}
case label:
case Label:
if !searchFuzzy(item.Labels, "", v) {
return false
}
......
......@@ -66,7 +66,7 @@ func (*nodeSearcher) fuzzy(fuzzy map[string]string, item *v1.Node) bool {
if !strings.Contains(item.Name, v) && !strings.Contains(item.Annotations[constants.DisplayNameAnnotationKey], v) {
return false
}
case label:
case Label:
if !searchFuzzy(item.Labels, "", v) {
return false
}
......
......@@ -66,7 +66,7 @@ func (*persistentVolumeClaimSearcher) fuzzy(fuzzy map[string]string, item *v1.Pe
if !strings.Contains(item.Name, v) && !strings.Contains(item.Annotations[constants.DisplayNameAnnotationKey], v) {
return false
}
case label:
case Label:
if !searchFuzzy(item.Labels, "", v) {
return false
}
......
......@@ -18,7 +18,7 @@
package resources
import (
v12 "k8s.io/api/apps/v1"
appsv1 "k8s.io/api/apps/v1"
"kubesphere.io/kubesphere/pkg/constants"
"kubesphere.io/kubesphere/pkg/informers"
"kubesphere.io/kubesphere/pkg/params"
......@@ -63,7 +63,7 @@ func podBelongTo(item *v1.Pod, kind string, name string) bool {
return false
}
func replicaSetBelongToDeployment(replicaSet *v12.ReplicaSet, deploymentName string) bool {
func replicaSetBelongToDeployment(replicaSet *appsv1.ReplicaSet, deploymentName string) bool {
for _, owner := range replicaSet.OwnerReferences {
if owner.Kind == "Deployment" && owner.Name == deploymentName {
return true
......@@ -150,11 +150,11 @@ func podBelongToService(item *v1.Pod, serviceName string) bool {
func (*podSearcher) match(match map[string]string, item *v1.Pod) bool {
for k, v := range match {
switch k {
case ownerKind:
case OwnerKind:
fallthrough
case ownerName:
kind := match[ownerKind]
name := match[ownerName]
case OwnerName:
kind := match[OwnerKind]
name := match[OwnerName]
if !podBelongTo(item, kind, name) {
return false
}
......@@ -196,7 +196,7 @@ func (*podSearcher) fuzzy(fuzzy map[string]string, item *v1.Pod) bool {
if !strings.Contains(item.Name, v) && !strings.Contains(item.Annotations[constants.DisplayNameAnnotationKey], v) {
return false
}
case label:
case Label:
if !searchFuzzy(item.Labels, "", v) {
return false
}
......
......@@ -19,6 +19,7 @@ package resources
import (
"fmt"
"github.com/golang/glog"
"kubesphere.io/kubesphere/pkg/models"
"kubesphere.io/kubesphere/pkg/params"
"kubesphere.io/kubesphere/pkg/utils/sliceutil"
......@@ -56,12 +57,12 @@ var (
const (
Name = "name"
label = "label"
ownerKind = "ownerKind"
ownerName = "ownerName"
Label = "label"
OwnerKind = "ownerKind"
OwnerName = "ownerName"
CreateTime = "createTime"
updateTime = "updateTime"
lastScheduleTime = "lastScheduleTime"
UpdateTime = "updateTime"
LastScheduleTime = "lastScheduleTime"
chart = "chart"
release = "release"
annotation = "annotation"
......@@ -106,6 +107,7 @@ func GetResource(namespace, resource, name string) (interface{}, error) {
if searcher, ok := resources[resource]; ok {
resource, err := searcher.get(namespace, name)
if err != nil {
glog.Errorln("get resource", namespace, resource, name, err)
return nil, err
}
return resource, nil
......@@ -120,16 +122,19 @@ func ListResources(namespace, resource string, conditions *params.Conditions, or
// none namespace resource
if namespace != "" && sliceutil.HasString(clusterResources, resource) {
glog.Errorln("resources not found", resource)
return nil, fmt.Errorf("not found")
}
if searcher, ok := resources[resource]; ok {
result, err = searcher.search(namespace, conditions, orderBy, reverse)
} else {
glog.Errorln("resources not found", resource)
return nil, fmt.Errorf("not found")
}
if err != nil {
glog.Errorln("resources search", err)
return nil, err
}
......
......@@ -66,7 +66,7 @@ func (*roleSearcher) fuzzy(fuzzy map[string]string, item *rbac.Role) bool {
if !strings.Contains(item.Name, v) && !strings.Contains(item.Annotations[constants.DisplayNameAnnotationKey], v) {
return false
}
case label:
case Label:
if !searchFuzzy(item.Labels, "", v) {
return false
}
......
......@@ -66,7 +66,7 @@ func (*s2iBuilderSearcher) fuzzy(fuzzy map[string]string, item *v1alpha1.S2iBuil
if !strings.Contains(item.Name, v) && !strings.Contains(item.Annotations[constants.DisplayNameAnnotationKey], v) {
return false
}
case label:
case Label:
if !searchFuzzy(item.Labels, "", v) {
return false
}
......
......@@ -66,7 +66,7 @@ func (*s2iBuilderTemplateSearcher) fuzzy(fuzzy map[string]string, item *v1alpha1
if !strings.Contains(item.Name, v) && !strings.Contains(item.Annotations[constants.DisplayNameAnnotationKey], v) {
return false
}
case label:
case Label:
if !searchFuzzy(item.Labels, "", v) {
return false
}
......
......@@ -73,7 +73,7 @@ func (*s2iRunSearcher) fuzzy(fuzzy map[string]string, item *v1alpha1.S2iRun) boo
if !strings.Contains(item.Name, v) && !strings.Contains(item.Annotations[constants.DisplayNameAnnotationKey], v) {
return false
}
case label:
case Label:
if !searchFuzzy(item.Labels, "", v) {
return false
}
......
......@@ -70,7 +70,7 @@ func (*secretSearcher) fuzzy(fuzzy map[string]string, item *v1.Secret) bool {
if !strings.Contains(item.Name, v) && !strings.Contains(item.Annotations[constants.DisplayNameAnnotationKey], v) {
return false
}
case label:
case Label:
if !searchFuzzy(item.Labels, "", v) {
return false
}
......
......@@ -66,7 +66,7 @@ func (*serviceSearcher) fuzzy(fuzzy map[string]string, item *v1.Service) bool {
if !strings.Contains(item.Name, v) && !strings.Contains(item.Annotations[constants.DisplayNameAnnotationKey], v) {
return false
}
case label:
case Label:
if !searchFuzzy(item.Labels, "", v) {
return false
}
......
......@@ -83,7 +83,7 @@ func (*statefulSetSearcher) fuzzy(fuzzy map[string]string, item *v1.StatefulSet)
if !strings.Contains(item.Name, v) && !strings.Contains(item.Annotations[constants.DisplayNameAnnotationKey], v) {
return false
}
case label:
case Label:
if !searchFuzzy(item.Labels, "", v) {
return false
}
......
......@@ -66,7 +66,7 @@ func (*storageClassesSearcher) fuzzy(fuzzy map[string]string, item *v1.StorageCl
if !strings.Contains(item.Name, v) && !strings.Contains(item.Annotations[constants.DisplayNameAnnotationKey], v) {
return false
}
case label:
case Label:
if !searchFuzzy(item.Labels, "", v) {
return false
}
......
......@@ -66,7 +66,7 @@ func (*workspaceSearcher) fuzzy(fuzzy map[string]string, item *tenantv1alpha1.Wo
if !strings.Contains(item.Name, v) && !strings.Contains(item.Annotations[constants.DisplayNameAnnotationKey], v) {
return false
}
case label:
case Label:
if !searchFuzzy(item.Labels, "", v) {
return false
}
......
......@@ -18,7 +18,7 @@
package models
import (
v12 "k8s.io/api/core/v1"
corev1 "k8s.io/api/core/v1"
"time"
"k8s.io/api/rbac/v1"
......@@ -117,6 +117,6 @@ type Token struct {
}
type ResourceQuota struct {
Namespace string `json:"namespace"`
Data v12.ResourceQuotaStatus `json:"data"`
Namespace string `json:"namespace"`
Data corev1.ResourceQuotaStatus `json:"data"`
}
......@@ -144,11 +144,10 @@ func CreateWorkspaceRoleBinding(workspace, username string, role string) error {
roleBindingName := fmt.Sprintf("workspace:%s:%s", workspace, strings.TrimPrefix(role, "workspace-"))
workspaceRoleBinding, err := informers.SharedInformerFactory().Rbac().V1().ClusterRoleBindings().Lister().Get(roleBindingName)
workspaceRoleBinding = workspaceRoleBinding.DeepCopy()
if err != nil {
return err
}
workspaceRoleBinding = workspaceRoleBinding.DeepCopy()
if !k8sutil.ContainsUser(workspaceRoleBinding.Subjects, username) {
workspaceRoleBinding.Subjects = append(workspaceRoleBinding.Subjects, v1.Subject{APIGroup: "rbac.authorization.k8s.io", Kind: "User", Name: username})
_, err = k8s.Client().RbacV1().ClusterRoleBindings().Update(workspaceRoleBinding)
......
......@@ -20,6 +20,7 @@ package ldap
import (
"flag"
"github.com/go-ldap/ldap"
"github.com/golang/glog"
"log"
"sync"
)
......@@ -67,6 +68,7 @@ func Client() (ldap.Client, error) {
conn, err := ldapClientPool().Get()
if err != nil {
glog.Errorln("get ldap connection from pool", err)
return nil, err
}
......@@ -74,6 +76,7 @@ func Client() (ldap.Client, error) {
if err != nil {
conn.Close()
glog.Errorln("bind manager dn", err)
return nil, err
}
......
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册