Merge pull request #1985 from zheng1/sdn

Add network topology APIs

cmd/ks-apiserver/app/options/options.go

@@ -17,6 +17,7 @@ import (
 	"kubesphere.io/kubesphere/pkg/simple/client/ldap"
 	esclient "kubesphere.io/kubesphere/pkg/simple/client/logging/elasticsearch"
 	"kubesphere.io/kubesphere/pkg/simple/client/monitoring/prometheus"
+	"kubesphere.io/kubesphere/pkg/simple/client/network"
 	"kubesphere.io/kubesphere/pkg/simple/client/openpitrix"
 	"kubesphere.io/kubesphere/pkg/simple/client/s3"
 	fakes3 "kubesphere.io/kubesphere/pkg/simple/client/s3/fake"
@@ -43,6 +44,7 @@ func NewServerRunOptions() *ServerRunOptions {
 			DevopsOptions:         jenkins.NewDevopsOptions(),
 			SonarQubeOptions:      sonarqube.NewSonarQubeOptions(),
 			ServiceMeshOptions:    servicemesh.NewServiceMeshOptions(),
+			NetworkOptions:        network.NewNetworkOptions(),
 			MonitoringOptions:     prometheus.NewPrometheusOptions(),
 			S3Options:             s3.NewS3Options(),
 			OpenPitrixOptions:     openpitrix.NewOptions(),
@@ -68,6 +70,7 @@ func (s *ServerRunOptions) Flags() (fss cliflag.NamedFlagSets) {
 	s.RedisOptions.AddFlags(fss.FlagSet("redis"), s.RedisOptions)
 	s.S3Options.AddFlags(fss.FlagSet("s3"), s.S3Options)
 	s.OpenPitrixOptions.AddFlags(fss.FlagSet("openpitrix"), s.OpenPitrixOptions)
+	s.NetworkOptions.AddFlags(fss.FlagSet("network"), s.NetworkOptions)
 	s.ServiceMeshOptions.AddFlags(fss.FlagSet("servicemesh"), s.ServiceMeshOptions)
 	s.MonitoringOptions.AddFlags(fss.FlagSet("monitoring"), s.MonitoringOptions)
 	s.LoggingOptions.AddFlags(fss.FlagSet("logging"), s.LoggingOptions)

cmd/ks-apiserver/app/options/validation.go

@@ -13,6 +13,7 @@ func (s *ServerRunOptions) Validate() []error {
 	errors = append(errors, s.SonarQubeOptions.Validate()...)
 	errors = append(errors, s.S3Options.Validate()...)
 	errors = append(errors, s.OpenPitrixOptions.Validate()...)
+	errors = append(errors, s.NetworkOptions.Validate()...)
 	errors = append(errors, s.LoggingOptions.Validate()...)
 
 	return errors

pkg/apiserver/apiserver.go

@@ -30,6 +30,7 @@ import (
 	iamv1alpha2 "kubesphere.io/kubesphere/pkg/kapis/iam/v1alpha2"
 	loggingv1alpha2 "kubesphere.io/kubesphere/pkg/kapis/logging/v1alpha2"
 	monitoringv1alpha3 "kubesphere.io/kubesphere/pkg/kapis/monitoring/v1alpha3"
+	networkv1alpha2 "kubesphere.io/kubesphere/pkg/kapis/network/v1alpha2"
 	"kubesphere.io/kubesphere/pkg/kapis/oauth"
 	openpitrixv1 "kubesphere.io/kubesphere/pkg/kapis/openpitrix/v1"
 	operationsv1alpha2 "kubesphere.io/kubesphere/pkg/kapis/operations/v1alpha2"
@@ -139,6 +140,7 @@ func (s *APIServer) installKubeSphereAPIs() {
 	urlruntime.Must(loggingv1alpha2.AddToContainer(s.container, s.KubernetesClient, s.LoggingClient))
 	urlruntime.Must(monitoringv1alpha3.AddToContainer(s.container, s.KubernetesClient.Kubernetes(), s.MonitoringClient))
 	urlruntime.Must(openpitrixv1.AddToContainer(s.container, s.InformerFactory, s.OpenpitrixClient))
+	urlruntime.Must(networkv1alpha2.AddToContainer(s.container, s.Config.NetworkOptions.WeaveScopeHost))
 	urlruntime.Must(operationsv1alpha2.AddToContainer(s.container, s.KubernetesClient.Kubernetes()))
 	urlruntime.Must(resourcesv1alpha2.AddToContainer(s.container, s.KubernetesClient.Kubernetes(), s.InformerFactory))
 	urlruntime.Must(tenantv1alpha2.AddToContainer(s.container, s.KubernetesClient, s.InformerFactory))

pkg/apiserver/config/config.go

@@ -11,6 +11,7 @@ import (
 	"kubesphere.io/kubesphere/pkg/simple/client/ldap"
 	"kubesphere.io/kubesphere/pkg/simple/client/logging/elasticsearch"
 	"kubesphere.io/kubesphere/pkg/simple/client/monitoring/prometheus"
+	"kubesphere.io/kubesphere/pkg/simple/client/network"
 	"kubesphere.io/kubesphere/pkg/simple/client/notification"
 	"kubesphere.io/kubesphere/pkg/simple/client/openpitrix"
 	"kubesphere.io/kubesphere/pkg/simple/client/s3"
@@ -61,6 +62,7 @@ type Config struct {
 	SonarQubeOptions      *sonarqube.Options                 `json:"sonarqube,omitempty" yaml:"sonarQube,omitempty" mapstructure:"sonarqube"`
 	KubernetesOptions     *k8s.KubernetesOptions             `json:"kubernetes,omitempty" yaml:"kubernetes,omitempty" mapstructure:"kubernetes"`
 	ServiceMeshOptions    *servicemesh.Options               `json:"servicemesh,omitempty" yaml:"servicemesh,omitempty" mapstructure:"servicemesh"`
+	NetworkOptions        *network.Options                   `json:"network,omitempty" yaml:"network,omitempty" mapstructure:"network"`
 	LdapOptions           *ldap.Options                      `json:"ldap,omitempty" yaml:"ldap,omitempty" mapstructure:"ldap"`
 	RedisOptions          *cache.Options                     `json:"redis,omitempty" yaml:"redis,omitempty" mapstructure:"redis"`
 	S3Options             *s3.Options                        `json:"s3,omitempty" yaml:"s3,omitempty" mapstructure:"s3"`
@@ -81,6 +83,7 @@ func New() *Config {
 		SonarQubeOptions:      sonarqube.NewSonarQubeOptions(),
 		KubernetesOptions:     k8s.NewKubernetesOptions(),
 		ServiceMeshOptions:    servicemesh.NewServiceMeshOptions(),
+		NetworkOptions:        network.NewNetworkOptions(),
 		LdapOptions:           ldap.NewOptions(),
 		RedisOptions:          cache.NewRedisOptions(),
 		S3Options:             s3.NewS3Options(),
@@ -175,6 +178,10 @@ func (conf *Config) stripEmptyOptions() {
 		conf.OpenPitrixOptions = nil
 	}
 
+	if conf.NetworkOptions != nil && conf.NetworkOptions.WeaveScopeHost == "" {
+		conf.NetworkOptions = nil
+	}
+
 	if conf.ServiceMeshOptions != nil && conf.ServiceMeshOptions.IstioPilotHost == "" &&
 		conf.ServiceMeshOptions.ServicemeshPrometheusHost == "" &&
 		conf.ServiceMeshOptions.JaegerQueryHost == "" {

pkg/apiserver/config/config_test.go

@@ -14,6 +14,7 @@ import (
 	"kubesphere.io/kubesphere/pkg/simple/client/ldap"
 	"kubesphere.io/kubesphere/pkg/simple/client/logging/elasticsearch"
 	"kubesphere.io/kubesphere/pkg/simple/client/monitoring/prometheus"
+	"kubesphere.io/kubesphere/pkg/simple/client/network"
 	"kubesphere.io/kubesphere/pkg/simple/client/notification"
 	"kubesphere.io/kubesphere/pkg/simple/client/openpitrix"
 	"kubesphere.io/kubesphere/pkg/simple/client/s3"
@@ -79,6 +80,9 @@ func newTestConfig() (*Config, error) {
 			CategoryManagerEndpoint:   "openpitrix-hyperpitrix.openpitrix-system.svc:9113",
 			AttachmentManagerEndpoint: "openpitrix-hyperpitrix.openpitrix-system.svc:9122",
 		},
+		NetworkOptions: &network.Options{
+			WeaveScopeHost: "weave-scope-app.weave.svc",
+		},
 		MonitoringOptions: &prometheus.Options{
 			Endpoint:          "http://prometheus.kubesphere-monitoring-system.svc",
 			SecondaryEndpoint: "http://prometheus.kubesphere-monitoring-system.svc",

pkg/constants/constants.go

@@ -59,6 +59,7 @@ const (
 	OpenpitrixTag              = "Openpitrix Resources"
 	VerificationTag            = "Verification"
 	RegistryTag                = "Docker Registry"
+	NetworkTopologyTag         = "Network Topology"
 	UserResourcesTag           = "User Resources"
 	DevOpsProjectTag           = "DevOps Project"
 	DevOpsProjectCredentialTag = "DevOps Project Credential"

pkg/kapis/network/group.go

+/*
+Copyright 2020 The KubeSphere authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package network contains network API versions
+package network

pkg/kapis/network/v1alpha2/handler.go

+package v1alpha2
+
+import (
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"net/url"
+
+	"github.com/emicklei/go-restful"
+	"k8s.io/klog"
+
+	"kubesphere.io/kubesphere/pkg/api"
+)
+
+const ScopeQueryUrl = "http://%s/api/topology/services"
+
+type handler struct {
+	weaveScopeHost string
+}
+
+func (h *handler) getScopeUrl() string {
+	return fmt.Sprintf(ScopeQueryUrl, h.weaveScopeHost)
+}
+
+func (h *handler) getNamespaceTopology(request *restful.Request, response *restful.Response) {
+	var query = url.Values{
+		"namespace": []string{request.PathParameter("namespace")},
+		"timestamp": request.QueryParameters("timestamp"),
+	}
+	var u = fmt.Sprintf("%s?%s", h.getScopeUrl(), query.Encode())
+
+	resp, err := http.Get(u)
+
+	if err != nil {
+		klog.Errorf("query scope faile with err %v", err)
+		api.HandleInternalError(response, nil, err)
+		return
+	}
+
+	body, err := ioutil.ReadAll(resp.Body)
+	defer resp.Body.Close()
+	if err != nil {
+		klog.Errorf("read response error : %v", err)
+		api.HandleInternalError(response, nil, err)
+		return
+	}
+
+	// need to set header for proper response
+	response.Header().Set("Content-Type", "application/json")
+	_, err = response.Write(body)
+
+	if err != nil {
+		klog.Errorf("write response failed %v", err)
+	}
+}
+
+func (h *handler) getNamespaceNodeTopology(request *restful.Request, response *restful.Response) {
+	var query = url.Values{
+		"namespace": []string{request.PathParameter("namespace")},
+		"timestamp": request.QueryParameters("timestamp"),
+	}
+	var u = fmt.Sprintf("%s/%s?%s", h.getScopeUrl(), request.PathParameter("node_id"), query.Encode())
+
+	resp, err := http.Get(u)
+
+	if err != nil {
+		klog.Errorf("query scope faile with err %v", err)
+		api.HandleInternalError(response, nil, err)
+		return
+	}
+
+	body, err := ioutil.ReadAll(resp.Body)
+	defer resp.Body.Close()
+	if err != nil {
+		klog.Errorf("read response error : %v", err)
+		api.HandleInternalError(response, nil, err)
+		return
+	}
+
+	// need to set header for proper response
+	response.Header().Set("Content-Type", "application/json")
+	_, err = response.Write(body)
+
+	if err != nil {
+		klog.Errorf("write response failed %v", err)
+	}
+}

pkg/kapis/network/v1alpha2/register.go

+/*
+Copyright 2020 The KubeSphere authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package v1alpha2
+
+import (
+	"net/http"
+
+	restful "github.com/emicklei/go-restful"
+	restfulspec "github.com/emicklei/go-restful-openapi"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"kubesphere.io/kubesphere/pkg/apiserver/runtime"
+	"kubesphere.io/kubesphere/pkg/constants"
+)
+
+const GroupName = "network.kubesphere.io"
+
+var GroupVersion = schema.GroupVersion{Group: GroupName, Version: "v1alpha2"}
+
+func AddToContainer(c *restful.Container, weaveScopeHost string) error {
+	webservice := runtime.NewWebService(GroupVersion)
+	h := handler{weaveScopeHost: weaveScopeHost}
+
+	webservice.Route(webservice.GET("/namespaces/{namespace}/topology").
+		To(h.getNamespaceTopology).
+		Metadata(restfulspec.KeyOpenAPITags, []string{constants.NetworkTopologyTag}).
+		Doc("Get the topology with specifying a namespace").
+		Param(webservice.PathParameter("namespace", "name of the namespace").Required(true)).
+		Returns(http.StatusOK, "ok", TopologyResponse{}).
+		Writes(TopologyResponse{})).
+		Produces(restful.MIME_JSON)
+
+	webservice.Route(webservice.GET("/namespaces/{namespace}/topology/{node_id}").
+		To(h.getNamespaceNodeTopology).
+		Metadata(restfulspec.KeyOpenAPITags, []string{constants.NetworkTopologyTag}).
+		Doc("Get the topology with specifying a node id in the whole topology and specifying a namespace").
+		Param(webservice.PathParameter("namespace", "name of the namespace").Required(true)).
+		Param(webservice.PathParameter("node_id", "id of the node in the whole topology").Required(true)).
+		Returns(http.StatusOK, "ok", NodeResponse{}).
+		Writes(NodeResponse{})).
+		Produces(restful.MIME_JSON)
+
+	c.Add(webservice)
+
+	return nil
+}

pkg/kapis/network/v1alpha2/swagger-doc.go

+/*
+Copyright 2020 The KubeSphere authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package v1alpha2
+
+import (
+	"time"
+)
+
+/////////////////////
+// SWAGGER RESPONSES
+/////////////////////
+
+// NoContent: the response is empty
+type NoContent struct {
+	Status int32 `json:"status"`
+	Reason error `json:"reason"`
+}
+
+// BadRequestError: the client request is incorrect
+type BadRequestError struct {
+	Status int32 `json:"status"`
+	Reason error `json:"reason"`
+}
+
+// NotFoundError is the error message that is generated when server could not find
+// what was requested
+type NotFoundError struct {
+	Status int32 `json:"status"`
+	Reason error `json:"reason"`
+}
+
+// copy from github.com/weaveworks/scope v1.12.0
+
+// MetadataRow is a row for the metadata table.
+type MetadataRow struct {
+	ID       string  `json:"id"`
+	Label    string  `json:"label"`
+	Value    string  `json:"value"`
+	Priority float64 `json:"priority,omitempty"`
+	Datatype string  `json:"dataType,omitempty"`
+	Truncate int     `json:"truncate,omitempty"`
+}
+
+// BasicNodeSummary is basic summary information about a Node,
+// sufficient for rendering links to the node.
+type BasicNodeSummary struct {
+	ID         string `json:"id"`
+	Label      string `json:"label"`
+	LabelMinor string `json:"labelMinor"`
+	Rank       string `json:"rank"`
+	Shape      string `json:"shape,omitempty"`
+	Tag        string `json:"tag,omitempty"`
+	Stack      bool   `json:"stack,omitempty"`
+	Pseudo     bool   `json:"pseudo,omitempty"`
+}
+
+// Parent is the information needed to build a link to the parent of a Node.
+type Parent struct {
+	ID         string `json:"id"`
+	Label      string `json:"label"`
+	TopologyID string `json:"topologyId"`
+}
+
+// Metric is a list of timeseries data with some metadata. Clients must use the
+// Add method to add values.  Metrics are immutable.
+type Metric struct {
+	Samples []Sample `json:"samples,omitempty"`
+	Min     float64  `json:"min"`
+	Max     float64  `json:"max"`
+}
+
+func (m Metric) first() time.Time { return m.Samples[0].Timestamp }
+func (m Metric) last() time.Time  { return m.Samples[len(m.Samples)-1].Timestamp }
+
+// Sample is a single datapoint of a metric.
+type Sample struct {
+	Timestamp time.Time `json:"date"`
+	Value     float64   `json:"value"`
+}
+
+// MetricRow is a tuple of data used to render a metric as a sparkline and
+// accoutrements.
+type MetricRow struct {
+	ID         string
+	Label      string
+	Format     string
+	Group      string
+	Value      float64
+	ValueEmpty bool
+	Priority   float64
+	URL        string
+	Metric     *Metric
+}
+
+// NodeSummaryGroup is a topology-typed group of children for a Node.
+type NodeSummaryGroup struct {
+	ID         string        `json:"id"`
+	Label      string        `json:"label"`
+	Nodes      []NodeSummary `json:"nodes"`
+	TopologyID string        `json:"topologyId"`
+	Columns    []Column      `json:"columns"`
+}
+
+// Connection is a row in the connections table.
+type Connection struct {
+	ID         string        `json:"id"`     // ID of this element in the UI.  Must be unique for a given ConnectionsSummary.
+	NodeID     string        `json:"nodeId"` // ID of a node in the topology. Optional, must be set if linkable is true.
+	Label      string        `json:"label"`
+	LabelMinor string        `json:"labelMinor,omitempty"`
+	Metadata   []MetadataRow `json:"metadata,omitempty"`
+}
+
+// ConnectionsSummary is the table of connection to/form a node
+type ConnectionsSummary struct {
+	ID          string       `json:"id"`
+	TopologyID  string       `json:"topologyId"`
+	Label       string       `json:"label"`
+	Columns     []Column     `json:"columns"`
+	Connections []Connection `json:"connections"`
+}
+
+// Column is the type for multi-column tables in the UI.
+type Column struct {
+	ID       string `json:"id"`
+	Label    string `json:"label"`
+	DataType string `json:"dataType"`
+}
+
+// Row is the type that holds the table data for the UI. Entries map from column ID to cell value.
+type Row struct {
+	ID      string            `json:"id"`
+	Entries map[string]string `json:"entries"`
+}
+
+// Table is the type for a table in the UI.
+type Table struct {
+	ID              string   `json:"id"`
+	Label           string   `json:"label"`
+	Type            string   `json:"type"`
+	Columns         []Column `json:"columns"`
+	Rows            []Row    `json:"rows"`
+	TruncationCount int      `json:"truncationCount,omitempty"`
+}
+
+// StringSet is a sorted set of unique strings. Clients must use the Add
+// method to add strings.
+type StringSet []string
+
+// IDList is a list of string IDs, which are always sorted and unique.
+type IDList StringSet
+
+// NodeSummary is summary information about a Node.
+type NodeSummary struct {
+	BasicNodeSummary
+	Metadata  []MetadataRow `json:"metadata,omitempty"`
+	Parents   []Parent      `json:"parents,omitempty"`
+	Metrics   []MetricRow   `json:"metrics,omitempty"`
+	Tables    []Table       `json:"tables,omitempty"`
+	Adjacency IDList        `json:"adjacency,omitempty"`
+}
+
+type NodeSummaries map[string]NodeSummary
+
+type APITopology struct {
+	Nodes NodeSummaries `json:"nodes"`
+}
+
+// A Control basically describes an RPC
+type Control struct {
+	ID           string `json:"id"`
+	Human        string `json:"human"`
+	Icon         string `json:"icon"` // from https://fortawesome.github.io/Font-Awesome/cheatsheet/ please
+	Confirmation string `json:"confirmation,omitempty"`
+	Rank         int    `json:"rank"`
+}
+
+// ControlInstance contains a control description, and all the info
+// needed to execute it.
+type ControlInstance struct {
+	ProbeID string
+	NodeID  string
+	Control Control
+}
+
+// Node is the data type that's yielded to the JavaScript layer when
+// we want deep information about an individual node.
+type Node struct {
+	NodeSummary
+	Controls    []ControlInstance    `json:"controls"`
+	Children    []NodeSummaryGroup   `json:"children,omitempty"`
+	Connections []ConnectionsSummary `json:"connections,omitempty"`
+}
+
+type APINode struct {
+	Node Node `json:"node"`
+}
+
+type TopologyResponse struct {
+	APITopology
+}
+
+type NodeResponse struct {
+	APINode
+}

pkg/simple/client/network/options.go

+package network
+
+import "github.com/spf13/pflag"
+
+type Options struct {
+
+	// weave scope service host
+	WeaveScopeHost string `json:"weaveScopeHost,omitempty" yaml:"weaveScopeHost"`
+}
+
+// NewNetworkOptions returns a `zero` instance
+func NewNetworkOptions() *Options {
+	return &Options{
+		WeaveScopeHost: "weave-scope-app.weave.svc",
+	}
+}
+
+func (s *Options) Validate() []error {
+	var errors []error
+	return errors
+}
+
+func (s *Options) ApplyTo(options *Options) {
+	if s.WeaveScopeHost != "" {
+		options.WeaveScopeHost = s.WeaveScopeHost
+	}
+}
+
+func (s *Options) AddFlags(fs *pflag.FlagSet, c *Options) {
+	fs.StringVar(&s.WeaveScopeHost, "weave-scope-host", c.WeaveScopeHost, ""+
+		"weave scope service host")
+}

tools/cmd/doc-gen/main.go

@@ -40,6 +40,7 @@ import (
 	iamv1alpha2 "kubesphere.io/kubesphere/pkg/kapis/iam/v1alpha2"
 	loggingv1alpha2 "kubesphere.io/kubesphere/pkg/kapis/logging/v1alpha2"
 	monitoringv1alpha3 "kubesphere.io/kubesphere/pkg/kapis/monitoring/v1alpha3"
+	networkv1alpha2 "kubesphere.io/kubesphere/pkg/kapis/network/v1alpha2"
 	openpitrixv1 "kubesphere.io/kubesphere/pkg/kapis/openpitrix/v1"
 	operationsv1alpha2 "kubesphere.io/kubesphere/pkg/kapis/operations/v1alpha2"
 	resourcesv1alpha2 "kubesphere.io/kubesphere/pkg/kapis/resources/v1alpha2"
@@ -122,6 +123,7 @@ func generateSwaggerJson() []byte {
 	urlruntime.Must(tenantv1alpha2.AddToContainer(container, clientsets, informerFactory))
 	urlruntime.Must(terminalv1alpha2.AddToContainer(container, clientsets.Kubernetes(), nil))
 	urlruntime.Must(metricsv1alpha2.AddToContainer(container))
+	urlruntime.Must(networkv1alpha2.AddToContainer(container, ""))
 
 	config := restfulspec.Config{
 		WebServices:                   container.RegisteredWebServices(),