Merge pull request #15 from chris-sun-star/dev

add mysqld exporter

go.mod

@@ -8,7 +8,7 @@ require (
 	github.com/fsnotify/fsnotify v1.4.9 // indirect
 	github.com/gin-contrib/pprof v1.3.0
 	github.com/gin-gonic/gin v1.7.2
-	github.com/go-kit/kit v0.9.0
+	github.com/go-kit/kit v0.10.0
 	github.com/go-kit/log v0.1.0
 	github.com/go-sql-driver/mysql v1.6.0
 	github.com/golang/protobuf v1.5.2 // indirect
@@ -24,6 +24,7 @@ require (
 	github.com/prometheus/client_golang v1.11.0
 	github.com/prometheus/client_model v0.2.0
 	github.com/prometheus/common v0.30.0
+	github.com/prometheus/mysqld_exporter v0.13.0 // indirect
 	github.com/prometheus/node_exporter v1.2.2
 	github.com/sirupsen/logrus v1.8.1
 	github.com/smartystreets/goconvey v1.6.4

metric/prometheus.go

+// Copyright (c) 2021 OceanBase
+// obagent is licensed under Mulan PSL v2.
+// You can use this software according to the terms and conditions of the Mulan PSL v2.
+// You may obtain a copy of Mulan PSL v2 at:
+//
+// http://license.coscl.org.cn/MulanPSL2
+//
+// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+// EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+// MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+// See the Mulan PSL v2 for more details.
+
+package metric
+
+import (
+	"fmt"
+	"math"
+	"time"
+
+	dto "github.com/prometheus/client_model/go"
+)
+
+func ParseFromMetricFamily(metricFamily *dto.MetricFamily) []Metric {
+	var metrics []Metric
+	now := time.Now()
+	for _, m := range metricFamily.Metric {
+		tags := makeLabels(m, nil)
+		var fields map[string]interface{}
+
+		switch metricFamily.GetType() {
+
+		case dto.MetricType_SUMMARY:
+			fields = makeQuantiles(m)
+			fields["count"] = float64(m.GetSummary().GetSampleCount())
+			fields["sum"] = m.GetSummary().GetSampleSum()
+
+		case dto.MetricType_HISTOGRAM:
+			fields = makeBuckets(m)
+			fields["count"] = float64(m.GetHistogram().GetSampleCount())
+			fields["sum"] = m.GetHistogram().GetSampleSum()
+
+		default:
+			fields = getNameAndValue(m)
+		}
+
+		if len(fields) > 0 {
+			var t time.Time
+			if m.TimestampMs != nil && *m.TimestampMs > 0 {
+				t = time.Unix(0, *m.TimestampMs*1000000)
+			} else {
+				t = now
+			}
+			newMetric := NewMetric(metricFamily.GetName(), fields, tags, t, valueType(metricFamily.GetType()))
+			metrics = append(metrics, newMetric)
+		}
+	}
+	return metrics
+}
+
+func makeLabels(m *dto.Metric, defaultTags map[string]string) map[string]string {
+	result := map[string]string{}
+
+	for key, value := range defaultTags {
+		result[key] = value
+	}
+
+	for _, lp := range m.Label {
+		result[lp.GetName()] = lp.GetValue()
+	}
+
+	return result
+}
+
+func makeQuantiles(m *dto.Metric) map[string]interface{} {
+	fields := make(map[string]interface{})
+	for _, q := range m.GetSummary().Quantile {
+		if !math.IsNaN(q.GetValue()) {
+			fields[fmt.Sprint(q.GetQuantile())] = q.GetValue()
+		}
+	}
+	return fields
+}
+
+func makeBuckets(m *dto.Metric) map[string]interface{} {
+	fields := make(map[string]interface{})
+	for _, b := range m.GetHistogram().Bucket {
+		fields[fmt.Sprint(b.GetUpperBound())] = float64(b.GetCumulativeCount())
+	}
+	return fields
+}
+
+func getNameAndValue(m *dto.Metric) map[string]interface{} {
+	fields := make(map[string]interface{})
+	if m.Gauge != nil {
+		if !math.IsNaN(m.GetGauge().GetValue()) {
+			fields["gauge"] = m.GetGauge().GetValue()
+		}
+	} else if m.Counter != nil {
+		if !math.IsNaN(m.GetCounter().GetValue()) {
+			fields["counter"] = m.GetCounter().GetValue()
+		}
+	} else if m.Untyped != nil {
+		if !math.IsNaN(m.GetUntyped().GetValue()) {
+			fields["value"] = m.GetUntyped().GetValue()
+		}
+	}
+	return fields
+}
+
+func valueType(metricType dto.MetricType) Type {
+	switch metricType {
+	case dto.MetricType_COUNTER:
+		return Counter
+	case dto.MetricType_GAUGE:
+		return Gauge
+	case dto.MetricType_SUMMARY:
+		return Summary
+	case dto.MetricType_HISTOGRAM:
+		return Histogram
+	default:
+		return Untyped
+	}
+}

plugins/inputs/all.go

@@ -23,6 +23,9 @@ func init() {
 	plugins.GetInputManager().Register("mysqlTableInput", func() plugins.Input {
 		return &mysql.TableInput{}
 	})
+	plugins.GetInputManager().Register("mysqldInput", func() plugins.Input {
+		return &mysql.MysqldExporter{}
+	})
 	plugins.GetInputManager().Register("nodeExporterInput", func() plugins.Input {
 		return &nodeexporter.NodeExporter{}
 	})

plugins/inputs/mysql/mysqld.go

+// Copyright (c) 2021 OceanBase
+// obagent is licensed under Mulan PSL v2.
+// You can use this software according to the terms and conditions of the Mulan PSL v2.
+// You may obtain a copy of Mulan PSL v2 at:
+//
+// http://license.coscl.org.cn/MulanPSL2
+//
+// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+// EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+// MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+// See the Mulan PSL v2 for more details.
+
+package mysql
+
+import (
+	"context"
+
+	log2 "github.com/go-kit/kit/log/logrus"
+	kitLog "github.com/go-kit/log"
+	log "github.com/sirupsen/logrus"
+
+	_ "github.com/go-sql-driver/mysql"
+	"github.com/pkg/errors"
+	"github.com/prometheus/client_golang/prometheus"
+	"gopkg.in/yaml.v3"
+
+	"github.com/prometheus/mysqld_exporter/collector"
+
+	"github.com/oceanbase/obagent/metric"
+)
+
+const mysqldSampleConfig = `
+`
+
+const mysqldDescription = `
+`
+
+var (
+	dsn string
+)
+
+// scrapers lists all possible collection methods and if they should be enabled by default.
+var scrapers = map[collector.Scraper]bool{
+	collector.ScrapeGlobalStatus{}:                        true,
+	collector.ScrapeGlobalVariables{}:                     true,
+	collector.ScrapeSlaveStatus{}:                         true,
+	collector.ScrapeProcesslist{}:                         false,
+	collector.ScrapeUser{}:                                false,
+	collector.ScrapeTableSchema{}:                         false,
+	collector.ScrapeInfoSchemaInnodbTablespaces{}:         false,
+	collector.ScrapeInnodbMetrics{}:                       false,
+	collector.ScrapeAutoIncrementColumns{}:                false,
+	collector.ScrapeBinlogSize{}:                          false,
+	collector.ScrapePerfTableIOWaits{}:                    false,
+	collector.ScrapePerfIndexIOWaits{}:                    false,
+	collector.ScrapePerfTableLockWaits{}:                  false,
+	collector.ScrapePerfEventsStatements{}:                false,
+	collector.ScrapePerfEventsStatementsSum{}:             false,
+	collector.ScrapePerfEventsWaits{}:                     false,
+	collector.ScrapePerfFileEvents{}:                      false,
+	collector.ScrapePerfFileInstances{}:                   false,
+	collector.ScrapePerfMemoryEvents{}:                    false,
+	collector.ScrapePerfReplicationGroupMembers{}:         false,
+	collector.ScrapePerfReplicationGroupMemberStats{}:     false,
+	collector.ScrapePerfReplicationApplierStatsByWorker{}: false,
+	collector.ScrapeUserStat{}:                            false,
+	collector.ScrapeClientStat{}:                          false,
+	collector.ScrapeTableStat{}:                           false,
+	collector.ScrapeSchemaStat{}:                          false,
+	collector.ScrapeInnodbCmp{}:                           true,
+	collector.ScrapeInnodbCmpMem{}:                        true,
+	collector.ScrapeQueryResponseTime{}:                   true,
+	collector.ScrapeEngineTokudbStatus{}:                  false,
+	collector.ScrapeEngineInnodbStatus{}:                  false,
+	collector.ScrapeHeartbeat{}:                           false,
+	collector.ScrapeSlaveHosts{}:                          false,
+	collector.ScrapeReplicaHost{}:                         false,
+}
+
+type MysqldConfig struct {
+	Dsn          string          `yaml:"dsn"`
+	ScraperFlags map[string]bool `yaml:"scraperFlags"`
+}
+
+type MysqldExporter struct {
+	Config          *MysqldConfig
+	logger          kitLog.Logger
+	registry        *prometheus.Registry
+	collector       *collector.Exporter
+	enabledScrapers []collector.Scraper
+}
+
+func (m *MysqldExporter) Close() error {
+	m.registry.Unregister(m.collector)
+	return nil
+}
+
+func (m *MysqldExporter) SampleConfig() string {
+	return mysqldSampleConfig
+}
+
+func (m *MysqldExporter) Description() string {
+	return mysqldDescription
+}
+
+func (m *MysqldExporter) Init(config map[string]interface{}) error {
+	var pluginConfig MysqldConfig
+
+	configBytes, err := yaml.Marshal(config)
+	if err != nil {
+		return errors.Wrap(err, "mysqld exporter encode config")
+	}
+
+	err = yaml.Unmarshal(configBytes, &pluginConfig)
+	if err != nil {
+		return errors.Wrap(err, "mysqld exporter decode config")
+	}
+
+	m.logger = log2.NewLogrusLogger(log.StandardLogger())
+
+	m.Config = &pluginConfig
+	log.Info("table input init with config", m.Config)
+
+	m.enabledScrapers = make([]collector.Scraper, 0, len(scrapers))
+
+	for scraper, enabledByDefault := range scrapers {
+		enabled, found := m.Config.ScraperFlags[scraper.Name()]
+		if (found && enabled) || (!found && enabledByDefault) {
+			m.enabledScrapers = append(m.enabledScrapers, scraper)
+		}
+	}
+
+	ctx := context.Background()
+	m.collector = collector.New(ctx, m.Config.Dsn, collector.NewMetrics(), m.enabledScrapers, m.logger)
+	m.registry = prometheus.NewRegistry()
+	err = m.registry.Register(m.collector)
+	if err != nil {
+		return errors.Wrap(err, "mysqld exporter register collector")
+	}
+
+	return err
+}
+
+func (m *MysqldExporter) Collect() ([]metric.Metric, error) {
+	// TODO parse metric families
+
+	var metrics []metric.Metric
+
+	metricFamilies, err := m.registry.Gather()
+	if err != nil {
+		return nil, errors.Wrap(err, "node exporter registry gather")
+	}
+	for _, metricFamily := range metricFamilies {
+		metricsFromMetricFamily := metric.ParseFromMetricFamily(metricFamily)
+		metrics = append(metrics, metricsFromMetricFamily...)
+	}
+
+	return metrics, nil
+}

plugins/inputs/nodeexporter/node_exporter.go

@@ -13,17 +13,13 @@
 package nodeexporter
 
 import (
-	"fmt"
-	"math"
 	"os"
 	"sync"
-	"time"
 
 	log2 "github.com/go-kit/kit/log/logrus"
 	kitLog "github.com/go-kit/log"
 	"github.com/pkg/errors"
 	"github.com/prometheus/client_golang/prometheus"
-	dto "github.com/prometheus/client_model/go"
 	"github.com/prometheus/node_exporter/collector"
 	log "github.com/sirupsen/logrus"
 	"gopkg.in/alecthomas/kingpin.v2"
@@ -212,110 +208,15 @@ func (n *NodeExporter) Collect() ([]metric.Metric, error) {
 	if err != nil {
 		return nil, errors.Wrap(err, "node exporter registry gather")
 	}
-	now := time.Now()
 	for _, metricFamily := range metricFamilies {
 
 		_, exist := n.definedMetricFamilyNameSet[metricFamily.GetName()]
 		if exist {
 
-			for _, m := range metricFamily.Metric {
-				tags := makeLabels(m, nil)
-				var fields map[string]interface{}
-
-				switch metricFamily.GetType() {
-
-				case dto.MetricType_SUMMARY:
-					fields = makeQuantiles(m)
-					fields["count"] = float64(m.GetSummary().GetSampleCount())
-					fields["sum"] = m.GetSummary().GetSampleSum()
-
-				case dto.MetricType_HISTOGRAM:
-					fields = makeBuckets(m)
-					fields["count"] = float64(m.GetHistogram().GetSampleCount())
-					fields["sum"] = m.GetHistogram().GetSampleSum()
-
-				default:
-					fields = getNameAndValue(m)
-				}
-
-				if len(fields) > 0 {
-					var t time.Time
-					if m.TimestampMs != nil && *m.TimestampMs > 0 {
-						t = time.Unix(0, *m.TimestampMs*1000000)
-					} else {
-						t = now
-					}
-					newMetric := metric.NewMetric(metricFamily.GetName(), fields, tags, t, ValueType(metricFamily.GetType()))
-					metrics = append(metrics, newMetric)
-				}
-			}
+			metricsFromMetricFamily := metric.ParseFromMetricFamily(metricFamily)
+			metrics = append(metrics, metricsFromMetricFamily...)
 		}
 	}
 
 	return metrics, nil
 }
-
-func makeLabels(m *dto.Metric, defaultTags map[string]string) map[string]string {
-	result := map[string]string{}
-
-	for key, value := range defaultTags {
-		result[key] = value
-	}
-
-	for _, lp := range m.Label {
-		result[lp.GetName()] = lp.GetValue()
-	}
-
-	return result
-}
-
-func makeQuantiles(m *dto.Metric) map[string]interface{} {
-	fields := make(map[string]interface{})
-	for _, q := range m.GetSummary().Quantile {
-		if !math.IsNaN(q.GetValue()) {
-			fields[fmt.Sprint(q.GetQuantile())] = q.GetValue()
-		}
-	}
-	return fields
-}
-
-func makeBuckets(m *dto.Metric) map[string]interface{} {
-	fields := make(map[string]interface{})
-	for _, b := range m.GetHistogram().Bucket {
-		fields[fmt.Sprint(b.GetUpperBound())] = float64(b.GetCumulativeCount())
-	}
-	return fields
-}
-
-func getNameAndValue(m *dto.Metric) map[string]interface{} {
-	fields := make(map[string]interface{})
-	if m.Gauge != nil {
-		if !math.IsNaN(m.GetGauge().GetValue()) {
-			fields["gauge"] = m.GetGauge().GetValue()
-		}
-	} else if m.Counter != nil {
-		if !math.IsNaN(m.GetCounter().GetValue()) {
-			fields["counter"] = m.GetCounter().GetValue()
-		}
-	} else if m.Untyped != nil {
-		if !math.IsNaN(m.GetUntyped().GetValue()) {
-			fields["value"] = m.GetUntyped().GetValue()
-		}
-	}
-	return fields
-}
-
-func ValueType(metricType dto.MetricType) metric.Type {
-	switch metricType {
-	case dto.MetricType_COUNTER:
-		return metric.Counter
-	case dto.MetricType_GAUGE:
-		return metric.Gauge
-	case dto.MetricType_SUMMARY:
-		return metric.Summary
-	case dto.MetricType_HISTOGRAM:
-		return metric.Histogram
-	default:
-		return metric.Untyped
-	}
-}

plugins/inputs/prometheus/prometheus.go

@@ -13,14 +13,11 @@
 package prometheus
 
 import (
-	"fmt"
-	"math"
 	"net/http"
 	"sync"
 	"time"
 
 	"github.com/pkg/errors"
-	dto "github.com/prometheus/client_model/go"
 	"github.com/prometheus/common/expfmt"
 	log "github.com/sirupsen/logrus"
 	"gopkg.in/yaml.v3"
@@ -123,110 +120,11 @@ func collect(client *http.Client, url string, metricsTotal *[]metric.Metric, wai
 	}
 
 	var metrics []metric.Metric
-	now := time.Now()
 	for _, metricFamily := range metricFamilies {
-
-		for _, m := range metricFamily.Metric {
-			tags := makeLabels(m, nil)
-			var fields map[string]interface{}
-
-			switch metricFamily.GetType() {
-
-			case dto.MetricType_SUMMARY:
-				fields = makeQuantiles(m)
-				fields["count"] = float64(m.GetSummary().GetSampleCount())
-				fields["sum"] = m.GetSummary().GetSampleSum()
-
-			case dto.MetricType_HISTOGRAM:
-				fields = makeBuckets(m)
-				fields["count"] = float64(m.GetHistogram().GetSampleCount())
-				fields["sum"] = m.GetHistogram().GetSampleSum()
-
-			default:
-				fields = getNameAndValue(m)
-			}
-
-			if len(fields) > 0 {
-				var t time.Time
-				if m.TimestampMs != nil && *m.TimestampMs > 0 {
-					t = time.Unix(0, *m.TimestampMs*1000000)
-				} else {
-					t = now
-				}
-				newMetric := metric.NewMetric(metricFamily.GetName(), fields, tags, t, ValueType(metricFamily.GetType()))
-
-				metrics = append(metrics, newMetric)
-			}
-
-		}
-
+		metricsFromMetricFamily := metric.ParseFromMetricFamily(metricFamily)
+		metrics = append(metrics, metricsFromMetricFamily...)
 	}
 	mutex.Lock()
 	*metricsTotal = append(*metricsTotal, metrics...)
 	mutex.Unlock()
 }
-
-func makeLabels(m *dto.Metric, defaultTags map[string]string) map[string]string {
-	result := map[string]string{}
-
-	for key, value := range defaultTags {
-		result[key] = value
-	}
-
-	for _, lp := range m.Label {
-		result[lp.GetName()] = lp.GetValue()
-	}
-
-	return result
-}
-
-func makeQuantiles(m *dto.Metric) map[string]interface{} {
-	fields := make(map[string]interface{})
-	for _, q := range m.GetSummary().Quantile {
-		if !math.IsNaN(q.GetValue()) {
-			fields[fmt.Sprint(q.GetQuantile())] = q.GetValue()
-		}
-	}
-	return fields
-}
-
-func makeBuckets(m *dto.Metric) map[string]interface{} {
-	fields := make(map[string]interface{})
-	for _, b := range m.GetHistogram().Bucket {
-		fields[fmt.Sprint(b.GetUpperBound())] = float64(b.GetCumulativeCount())
-	}
-	return fields
-}
-
-func getNameAndValue(m *dto.Metric) map[string]interface{} {
-	fields := make(map[string]interface{})
-	if m.Gauge != nil {
-		if !math.IsNaN(m.GetGauge().GetValue()) {
-			fields["gauge"] = m.GetGauge().GetValue()
-		}
-	} else if m.Counter != nil {
-		if !math.IsNaN(m.GetCounter().GetValue()) {
-			fields["counter"] = m.GetCounter().GetValue()
-		}
-	} else if m.Untyped != nil {
-		if !math.IsNaN(m.GetUntyped().GetValue()) {
-			fields["value"] = m.GetUntyped().GetValue()
-		}
-	}
-	return fields
-}
-
-func ValueType(metricType dto.MetricType) metric.Type {
-	switch metricType {
-	case dto.MetricType_COUNTER:
-		return metric.Counter
-	case dto.MetricType_GAUGE:
-		return metric.Gauge
-	case dto.MetricType_SUMMARY:
-		return metric.Summary
-	case dto.MetricType_HISTOGRAM:
-		return metric.Histogram
-	default:
-		return metric.Untyped
-	}
-}