2019-06-16 03:21:44 +08:00
|
|
|
package stats
|
2019-06-13 17:01:51 +08:00
|
|
|
|
2019-06-15 04:06:01 +08:00
|
|
|
import (
|
2019-06-16 12:46:55 +08:00
|
|
|
"fmt"
|
|
|
|
"os"
|
2020-05-15 12:08:34 +08:00
|
|
|
"strings"
|
2019-06-15 04:06:01 +08:00
|
|
|
"time"
|
|
|
|
|
|
|
|
"github.com/prometheus/client_golang/prometheus"
|
|
|
|
"github.com/prometheus/client_golang/prometheus/push"
|
2020-05-15 12:08:34 +08:00
|
|
|
|
|
|
|
"github.com/chrislusf/seaweedfs/weed/glog"
|
2019-06-15 04:06:01 +08:00
|
|
|
)
|
2019-06-13 17:01:51 +08:00
|
|
|
|
|
|
|
var (
|
2019-06-16 03:21:44 +08:00
|
|
|
FilerGather = prometheus.NewRegistry()
|
|
|
|
VolumeServerGather = prometheus.NewRegistry()
|
2019-06-14 15:54:56 +08:00
|
|
|
|
2019-06-16 03:21:44 +08:00
|
|
|
FilerRequestCounter = prometheus.NewCounterVec(
|
2019-06-13 17:01:51 +08:00
|
|
|
prometheus.CounterOpts{
|
|
|
|
Namespace: "SeaweedFS",
|
|
|
|
Subsystem: "filer",
|
|
|
|
Name: "request_total",
|
|
|
|
Help: "Counter of filer requests.",
|
|
|
|
}, []string{"type"})
|
|
|
|
|
2019-06-16 03:21:44 +08:00
|
|
|
FilerRequestHistogram = prometheus.NewHistogramVec(
|
2019-06-13 17:01:51 +08:00
|
|
|
prometheus.HistogramOpts{
|
|
|
|
Namespace: "SeaweedFS",
|
|
|
|
Subsystem: "filer",
|
|
|
|
Name: "request_seconds",
|
|
|
|
Help: "Bucketed histogram of filer request processing time.",
|
2019-06-14 15:54:56 +08:00
|
|
|
Buckets: prometheus.ExponentialBuckets(0.0001, 2, 24),
|
|
|
|
}, []string{"type"})
|
|
|
|
|
2019-06-23 03:23:25 +08:00
|
|
|
FilerStoreCounter = prometheus.NewCounterVec(
|
|
|
|
prometheus.CounterOpts{
|
|
|
|
Namespace: "SeaweedFS",
|
|
|
|
Subsystem: "filerStore",
|
|
|
|
Name: "request_total",
|
|
|
|
Help: "Counter of filer store requests.",
|
|
|
|
}, []string{"store", "type"})
|
|
|
|
|
|
|
|
FilerStoreHistogram = prometheus.NewHistogramVec(
|
|
|
|
prometheus.HistogramOpts{
|
|
|
|
Namespace: "SeaweedFS",
|
|
|
|
Subsystem: "filerStore",
|
|
|
|
Name: "request_seconds",
|
|
|
|
Help: "Bucketed histogram of filer store request processing time.",
|
|
|
|
Buckets: prometheus.ExponentialBuckets(0.0001, 2, 24),
|
|
|
|
}, []string{"store", "type"})
|
|
|
|
|
2019-06-16 03:21:44 +08:00
|
|
|
VolumeServerRequestCounter = prometheus.NewCounterVec(
|
2019-06-14 15:54:56 +08:00
|
|
|
prometheus.CounterOpts{
|
|
|
|
Namespace: "SeaweedFS",
|
|
|
|
Subsystem: "volumeServer",
|
|
|
|
Name: "request_total",
|
2019-06-24 15:26:03 +08:00
|
|
|
Help: "Counter of volume server requests.",
|
2019-06-14 15:54:56 +08:00
|
|
|
}, []string{"type"})
|
|
|
|
|
2019-06-16 03:21:44 +08:00
|
|
|
VolumeServerRequestHistogram = prometheus.NewHistogramVec(
|
2019-06-14 15:54:56 +08:00
|
|
|
prometheus.HistogramOpts{
|
|
|
|
Namespace: "SeaweedFS",
|
|
|
|
Subsystem: "volumeServer",
|
|
|
|
Name: "request_seconds",
|
2019-06-24 15:26:03 +08:00
|
|
|
Help: "Bucketed histogram of volume server request processing time.",
|
2019-06-14 15:54:56 +08:00
|
|
|
Buckets: prometheus.ExponentialBuckets(0.0001, 2, 24),
|
2019-06-13 17:01:51 +08:00
|
|
|
}, []string{"type"})
|
2019-06-15 04:06:01 +08:00
|
|
|
|
2019-06-18 12:02:50 +08:00
|
|
|
VolumeServerVolumeCounter = prometheus.NewGaugeVec(
|
2019-06-15 04:06:01 +08:00
|
|
|
prometheus.GaugeOpts{
|
|
|
|
Namespace: "SeaweedFS",
|
|
|
|
Subsystem: "volumeServer",
|
|
|
|
Name: "volumes",
|
2019-06-18 12:02:50 +08:00
|
|
|
Help: "Number of volumes or shards.",
|
|
|
|
}, []string{"collection", "type"})
|
2019-06-16 17:24:15 +08:00
|
|
|
|
2019-06-18 12:02:50 +08:00
|
|
|
VolumeServerMaxVolumeCounter = prometheus.NewGauge(
|
2019-06-16 17:24:15 +08:00
|
|
|
prometheus.GaugeOpts{
|
|
|
|
Namespace: "SeaweedFS",
|
|
|
|
Subsystem: "volumeServer",
|
2019-06-21 00:56:49 +08:00
|
|
|
Name: "max_volumes",
|
2019-06-18 12:02:50 +08:00
|
|
|
Help: "Maximum number of volumes.",
|
2019-06-16 17:24:15 +08:00
|
|
|
})
|
2019-06-17 12:56:41 +08:00
|
|
|
|
|
|
|
VolumeServerDiskSizeGauge = prometheus.NewGaugeVec(
|
2019-06-16 17:44:20 +08:00
|
|
|
prometheus.GaugeOpts{
|
|
|
|
Namespace: "SeaweedFS",
|
|
|
|
Subsystem: "volumeServer",
|
2019-06-17 12:56:41 +08:00
|
|
|
Name: "total_disk_size",
|
2019-06-16 17:44:20 +08:00
|
|
|
Help: "Actual disk size used by volumes.",
|
2019-06-17 12:56:41 +08:00
|
|
|
}, []string{"collection", "type"})
|
2019-06-13 17:01:51 +08:00
|
|
|
)
|
|
|
|
|
|
|
|
func init() {
|
2019-06-14 15:54:56 +08:00
|
|
|
|
2019-06-16 03:21:44 +08:00
|
|
|
FilerGather.MustRegister(FilerRequestCounter)
|
|
|
|
FilerGather.MustRegister(FilerRequestHistogram)
|
2019-06-23 03:23:25 +08:00
|
|
|
FilerGather.MustRegister(FilerStoreCounter)
|
|
|
|
FilerGather.MustRegister(FilerStoreHistogram)
|
2019-06-29 18:02:28 +08:00
|
|
|
FilerGather.MustRegister(prometheus.NewGoCollector())
|
2019-06-14 15:54:56 +08:00
|
|
|
|
2019-06-16 03:21:44 +08:00
|
|
|
VolumeServerGather.MustRegister(VolumeServerRequestCounter)
|
|
|
|
VolumeServerGather.MustRegister(VolumeServerRequestHistogram)
|
|
|
|
VolumeServerGather.MustRegister(VolumeServerVolumeCounter)
|
2019-06-18 12:02:50 +08:00
|
|
|
VolumeServerGather.MustRegister(VolumeServerMaxVolumeCounter)
|
2019-06-17 12:56:41 +08:00
|
|
|
VolumeServerGather.MustRegister(VolumeServerDiskSizeGauge)
|
2019-06-15 04:06:01 +08:00
|
|
|
|
|
|
|
}
|
|
|
|
|
2019-06-18 05:51:47 +08:00
|
|
|
func LoopPushingMetric(name, instance string, gatherer *prometheus.Registry, fnGetMetricsDest func() (addr string, intervalSeconds int)) {
|
2019-06-15 04:06:01 +08:00
|
|
|
|
2019-06-24 06:29:49 +08:00
|
|
|
if fnGetMetricsDest == nil {
|
|
|
|
return
|
|
|
|
}
|
|
|
|
|
2019-06-18 05:51:47 +08:00
|
|
|
addr, intervalSeconds := fnGetMetricsDest()
|
2019-06-16 03:21:44 +08:00
|
|
|
pusher := push.New(addr, name).Gatherer(gatherer).Grouping("instance", instance)
|
2019-06-18 05:51:47 +08:00
|
|
|
currentAddr := addr
|
2019-06-14 15:54:56 +08:00
|
|
|
|
2019-06-15 04:06:01 +08:00
|
|
|
for {
|
2019-06-18 05:51:47 +08:00
|
|
|
if currentAddr != "" {
|
|
|
|
err := pusher.Push()
|
2020-05-15 12:08:34 +08:00
|
|
|
if err != nil && !strings.HasPrefix(err.Error(), "unexpected status code 200") {
|
2019-06-18 05:51:47 +08:00
|
|
|
glog.V(0).Infof("could not push metrics to prometheus push gateway %s: %v", addr, err)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
if intervalSeconds <= 0 {
|
|
|
|
intervalSeconds = 15
|
2019-06-15 04:06:01 +08:00
|
|
|
}
|
|
|
|
time.Sleep(time.Duration(intervalSeconds) * time.Second)
|
2019-06-18 05:51:47 +08:00
|
|
|
addr, intervalSeconds = fnGetMetricsDest()
|
|
|
|
if currentAddr != addr {
|
|
|
|
pusher = push.New(addr, name).Gatherer(gatherer).Grouping("instance", instance)
|
|
|
|
currentAddr = addr
|
|
|
|
}
|
|
|
|
|
2019-06-15 04:06:01 +08:00
|
|
|
}
|
2019-06-13 17:01:51 +08:00
|
|
|
}
|
2019-06-16 12:46:55 +08:00
|
|
|
|
2020-03-02 14:13:47 +08:00
|
|
|
func SourceName(port uint32) string {
|
2019-06-16 12:46:55 +08:00
|
|
|
hostname, err := os.Hostname()
|
|
|
|
if err != nil {
|
|
|
|
return "unknown"
|
|
|
|
}
|
2019-06-17 12:57:32 +08:00
|
|
|
return fmt.Sprintf("%s:%d", hostname, port)
|
2019-06-16 12:46:55 +08:00
|
|
|
}
|