From 28a1f42962a3c22fa341d62e52ed014ae17c508f Mon Sep 17 00:00:00 2001 From: ginn13 Date: Thu, 4 Aug 2022 19:08:40 +0300 Subject: [PATCH] Helm chart monitoring improvements (#3406) --- .../seaweedfs-grafana-dashboard.json | 75 +++++++++++-------- k8s/helm_charts2/templates/filer-service.yaml | 6 ++ .../templates/filer-servicemonitor.yaml | 7 +- .../templates/filer-statefulset.yaml | 4 +- .../templates/master-service.yaml | 6 ++ .../templates/master-servicemonitor.yaml | 21 ++++++ .../templates/master-statefulset.yaml | 14 +++- k8s/helm_charts2/templates/s3-deployment.yaml | 4 + .../templates/s3-servicemonitor.yaml | 7 +- .../templates/volume-service.yaml | 2 +- .../templates/volume-servicemonitor.yaml | 7 +- .../templates/volume-statefulset.yaml | 8 +- k8s/helm_charts2/values.yaml | 1 + other/metrics/grafana_seaweedfs_k8s.json | 14 ++-- 14 files changed, 122 insertions(+), 54 deletions(-) create mode 100644 k8s/helm_charts2/templates/master-servicemonitor.yaml diff --git a/k8s/helm_charts2/dashboards/seaweedfs-grafana-dashboard.json b/k8s/helm_charts2/dashboards/seaweedfs-grafana-dashboard.json index d492a0695..3e5621e4e 100644 --- a/k8s/helm_charts2/dashboards/seaweedfs-grafana-dashboard.json +++ b/k8s/helm_charts2/dashboards/seaweedfs-grafana-dashboard.json @@ -1,14 +1,5 @@ { - "__inputs": [ - { - "name": "DS_PROMETHEUS-DEV", - "label": "prometheus-dev", - "description": "", - "type": "datasource", - "pluginId": "prometheus", - "pluginName": "Prometheus" - } - ], + "__inputs": [], "__requires": [ { "type": "grafana", @@ -33,7 +24,7 @@ "list": [ { "builtIn": 1, - "datasource": "${DS_PROMETHEUS-DEV}", + "datasource": "${DS_PROMETHEUS}", "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", @@ -61,7 +52,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "${DS_PROMETHEUS-DEV}", + "datasource": "${DS_PROMETHEUS}", "editable": true, "error": false, "fill": 1, @@ -151,7 +142,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "${DS_PROMETHEUS-DEV}", + "datasource": "${DS_PROMETHEUS}", "editable": true, "error": false, "fill": 1, @@ -247,7 +238,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "${DS_PROMETHEUS-DEV}", + "datasource": "${DS_PROMETHEUS}", "editable": true, "error": false, "fill": 1, @@ -343,7 +334,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "${DS_PROMETHEUS-DEV}", + "datasource": "${DS_PROMETHEUS}", "editable": true, "error": false, "fill": 0, @@ -448,7 +439,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "${DS_PROMETHEUS-DEV}", + "datasource": "${DS_PROMETHEUS}", "editable": true, "error": false, "fill": 1, @@ -538,7 +529,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "${DS_PROMETHEUS-DEV}", + "datasource": "${DS_PROMETHEUS}", "editable": true, "error": false, "fill": 1, @@ -628,7 +619,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "${DS_PROMETHEUS-DEV}", + "datasource": "${DS_PROMETHEUS}", "editable": true, "error": false, "fill": 1, @@ -718,7 +709,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "${DS_PROMETHEUS-DEV}", + "datasource": "${DS_PROMETHEUS}", "editable": true, "error": false, "fill": 0, @@ -811,7 +802,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "${DS_PROMETHEUS-DEV}", + "datasource": "${DS_PROMETHEUS}", "editable": true, "error": false, "fill": 0, @@ -943,7 +934,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "${DS_PROMETHEUS-DEV}", + "datasource": "${DS_PROMETHEUS}", "editable": true, "error": false, "fill": 1, @@ -1031,7 +1022,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "${DS_PROMETHEUS-DEV}", + "datasource": "${DS_PROMETHEUS}", "editable": true, "error": false, "fill": 1, @@ -1117,7 +1108,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "${DS_PROMETHEUS-DEV}", + "datasource": "${DS_PROMETHEUS}", "fill": 1, "id": 48, "legend": { @@ -1200,7 +1191,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "${DS_PROMETHEUS-DEV}", + "datasource": "${DS_PROMETHEUS}", "fill": 1, "id": 50, "legend": { @@ -1283,7 +1274,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "${DS_PROMETHEUS-DEV}", + "datasource": "${DS_PROMETHEUS}", "fill": 1, "id": 51, "legend": { @@ -1371,7 +1362,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "${DS_PROMETHEUS-DEV}", + "datasource": "${DS_PROMETHEUS}", "editable": true, "error": false, "fill": 1, @@ -1451,7 +1442,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "${DS_PROMETHEUS-DEV}", + "datasource": "${DS_PROMETHEUS}", "editable": true, "error": false, "fill": 1, @@ -1546,7 +1537,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "${DS_PROMETHEUS-DEV}", + "datasource": "${DS_PROMETHEUS}", "editable": true, "error": false, "fill": 1, @@ -1651,7 +1642,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "${DS_PROMETHEUS-DEV}", + "datasource": "${DS_PROMETHEUS}", "editable": true, "error": false, "fill": 1, @@ -1731,7 +1722,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "${DS_PROMETHEUS-DEV}", + "datasource": "${DS_PROMETHEUS}", "editable": true, "error": false, "fill": 1, @@ -1819,7 +1810,29 @@ "style": "dark", "tags": [], "templating": { - "list": [] + "list": [ + { + "current": { + "selected": true, + "text": "Prometheus", + "value": "Prometheus" + }, + "description": null, + "error": null, + "hide": 0, + "includeAll": false, + "label": "Datasource", + "multi": false, + "name": "DS_PROMETHEUS", + "options": [], + "query": "prometheus", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + } + ] }, "time": { "from": "now-30d", diff --git a/k8s/helm_charts2/templates/filer-service.yaml b/k8s/helm_charts2/templates/filer-service.yaml index 45035fc27..2a222bd61 100644 --- a/k8s/helm_charts2/templates/filer-service.yaml +++ b/k8s/helm_charts2/templates/filer-service.yaml @@ -20,6 +20,12 @@ spec: port: {{ .Values.filer.grpcPort }} targetPort: {{ .Values.filer.grpcPort }} protocol: TCP + {{- if .Values.filer.metricsPort }} + - name: "metrics" + port: {{ .Values.filer.metricsPort }} + targetPort: {{ .Values.filer.metricsPort }} + protocol: TCP + {{- end }} selector: app: {{ template "seaweedfs.name" . }} component: filer diff --git a/k8s/helm_charts2/templates/filer-servicemonitor.yaml b/k8s/helm_charts2/templates/filer-servicemonitor.yaml index ed45442dc..4a891463c 100644 --- a/k8s/helm_charts2/templates/filer-servicemonitor.yaml +++ b/k8s/helm_charts2/templates/filer-servicemonitor.yaml @@ -11,10 +11,11 @@ metadata: spec: endpoints: - interval: 30s - port: swfs-filer-metrics + port: metrics scrapeTimeout: 5s selector: - app: {{ template "seaweedfs.name" . }} - component: filer + matchLabels: + app: {{ template "seaweedfs.name" . }} + component: filer {{- end }} {{- end }} \ No newline at end of file diff --git a/k8s/helm_charts2/templates/filer-statefulset.yaml b/k8s/helm_charts2/templates/filer-statefulset.yaml index c36783650..689c1092c 100644 --- a/k8s/helm_charts2/templates/filer-statefulset.yaml +++ b/k8s/helm_charts2/templates/filer-statefulset.yaml @@ -110,7 +110,7 @@ spec: filer \ -port={{ .Values.filer.port }} \ {{- if .Values.filer.metricsPort }} - -metricsPort {{ .Values.filer.metricsPort }} \ + -metricsPort={{ .Values.filer.metricsPort }} \ {{- end }} {{- if .Values.filer.redirectOnRead }} -redirectOnRead \ @@ -193,6 +193,8 @@ spec: ports: - containerPort: {{ .Values.filer.port }} name: swfs-filer + - containerPort: {{ .Values.filer.metricsPort }} + name: metrics - containerPort: {{ .Values.filer.grpcPort }} #name: swfs-filer-grpc readinessProbe: diff --git a/k8s/helm_charts2/templates/master-service.yaml b/k8s/helm_charts2/templates/master-service.yaml index 0ce467538..77710220b 100644 --- a/k8s/helm_charts2/templates/master-service.yaml +++ b/k8s/helm_charts2/templates/master-service.yaml @@ -20,6 +20,12 @@ spec: port: {{ .Values.master.grpcPort }} targetPort: {{ .Values.master.grpcPort }} protocol: TCP + {{- if .Values.master.metricsPort }} + - name: "metrics" + port: {{ .Values.master.metricsPort }} + targetPort: {{ .Values.master.metricsPort }} + protocol: TCP + {{- end }} selector: app: {{ template "seaweedfs.name" . }} component: master diff --git a/k8s/helm_charts2/templates/master-servicemonitor.yaml b/k8s/helm_charts2/templates/master-servicemonitor.yaml new file mode 100644 index 000000000..64ee18035 --- /dev/null +++ b/k8s/helm_charts2/templates/master-servicemonitor.yaml @@ -0,0 +1,21 @@ +{{- if .Values.master.metricsPort }} +{{- if .Values.global.monitoring.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "seaweedfs.name" . }}-master + namespace: {{ .Release.Namespace }} + labels: + app: {{ template "seaweedfs.name" . }} + component: master +spec: + endpoints: + - interval: 30s + port: metrics + scrapeTimeout: 5s + selector: + matchLabels: + app: {{ template "seaweedfs.name" . }} + component: master +{{- end }} +{{- end }} \ No newline at end of file diff --git a/k8s/helm_charts2/templates/master-statefulset.yaml b/k8s/helm_charts2/templates/master-statefulset.yaml index 65038be78..0b58843fd 100644 --- a/k8s/helm_charts2/templates/master-statefulset.yaml +++ b/k8s/helm_charts2/templates/master-statefulset.yaml @@ -109,7 +109,14 @@ spec: -volumePreallocate \ {{- end }} {{- if .Values.global.monitoring.enabled }} + {{- if and .Values.global.monitoring.gatewayHost .Values.global.monitoring.gatewayPort }} -metrics.address="{{ .Values.global.monitoring.gatewayHost }}:{{ .Values.global.monitoring.gatewayPort }}" \ + {{- if .Values.master.metricsIntervalSec }} + -metrics.intervalSeconds={{ .Values.master.metricsIntervalSec }} \ + {{- end }} + {{- else if .Values.master.metricsPort }} + -metricsPort={{ .Values.master.metricsPort }} \ + {{- end }} {{- end }} -volumeSizeLimitMB={{ .Values.master.volumeSizeLimitMB }} \ {{- if .Values.master.disableHttp }} @@ -121,9 +128,6 @@ spec: {{- if .Values.master.garbageThreshold }} -garbageThreshold={{ .Values.master.garbageThreshold }} \ {{- end }} - {{- if .Values.master.metricsIntervalSec }} - -metrics.intervalSeconds={{ .Values.master.metricsIntervalSec }} \ - {{- end }} -ip=${POD_NAME}.${SEAWEEDFS_FULLNAME}-master \ -peers={{ range $index := until (.Values.master.replicas | int) }}${SEAWEEDFS_FULLNAME}-master-{{ $index }}.${SEAWEEDFS_FULLNAME}-master:{{ $.Values.master.port }}{{ if lt $index (sub ($.Values.master.replicas | int) 1) }},{{ end }}{{ end }} volumeMounts: @@ -156,6 +160,10 @@ spec: ports: - containerPort: {{ .Values.master.port }} name: swfs-master + {{- if and .Values.global.monitoring.enabled .Values.master.metricsPort }} + - containerPort: {{ .Values.master.metricsPort }} + name: metrics + {{- end }} - containerPort: {{ .Values.master.grpcPort }} #name: swfs-master-grpc readinessProbe: diff --git a/k8s/helm_charts2/templates/s3-deployment.yaml b/k8s/helm_charts2/templates/s3-deployment.yaml index e3b5b33b4..b9efef827 100644 --- a/k8s/helm_charts2/templates/s3-deployment.yaml +++ b/k8s/helm_charts2/templates/s3-deployment.yaml @@ -131,6 +131,10 @@ spec: ports: - containerPort: {{ .Values.s3.port }} name: swfs-s3 + {{- if .Values.s3.metricsPort }} + - containerPort: {{ .Values.s3.metricsPort }} + name: "metrics" + {{- end }} readinessProbe: httpGet: path: /status diff --git a/k8s/helm_charts2/templates/s3-servicemonitor.yaml b/k8s/helm_charts2/templates/s3-servicemonitor.yaml index b549893c7..5df537086 100644 --- a/k8s/helm_charts2/templates/s3-servicemonitor.yaml +++ b/k8s/helm_charts2/templates/s3-servicemonitor.yaml @@ -11,10 +11,11 @@ metadata: spec: endpoints: - interval: 30s - port: swfs-s3-metrics + port: metrics scrapeTimeout: 5s selector: - app: {{ template "seaweedfs.name" . }} - component: s3 + matchLabels: + app: {{ template "seaweedfs.name" . }} + component: s3 {{- end }} {{- end }} \ No newline at end of file diff --git a/k8s/helm_charts2/templates/volume-service.yaml b/k8s/helm_charts2/templates/volume-service.yaml index 0a9173fde..1e07865c0 100644 --- a/k8s/helm_charts2/templates/volume-service.yaml +++ b/k8s/helm_charts2/templates/volume-service.yaml @@ -18,7 +18,7 @@ spec: targetPort: {{ .Values.volume.grpcPort }} protocol: TCP {{- if .Values.volume.metricsPort }} - - name: "swfs-volume-metrics" + - name: "metrics" port: {{ .Values.volume.metricsPort }} targetPort: {{ .Values.volume.metricsPort }} protocol: TCP diff --git a/k8s/helm_charts2/templates/volume-servicemonitor.yaml b/k8s/helm_charts2/templates/volume-servicemonitor.yaml index 90d70e8de..2dcca5ac8 100644 --- a/k8s/helm_charts2/templates/volume-servicemonitor.yaml +++ b/k8s/helm_charts2/templates/volume-servicemonitor.yaml @@ -11,10 +11,11 @@ metadata: spec: endpoints: - interval: 30s - port: swfs-volume-metrics + port: metrics scrapeTimeout: 5s selector: - app: {{ template "seaweedfs.name" . }} - component: volume + matchLabels: + app: {{ template "seaweedfs.name" . }} + component: volume {{- end }} {{- end }} \ No newline at end of file diff --git a/k8s/helm_charts2/templates/volume-statefulset.yaml b/k8s/helm_charts2/templates/volume-statefulset.yaml index d0031e1e2..9770afed7 100644 --- a/k8s/helm_charts2/templates/volume-statefulset.yaml +++ b/k8s/helm_charts2/templates/volume-statefulset.yaml @@ -102,7 +102,7 @@ spec: volume \ -port={{ .Values.volume.port }} \ {{- if .Values.volume.metricsPort }} - -metricsPort {{ .Values.volume.metricsPort }} \ + -metricsPort={{ .Values.volume.metricsPort }} \ {{- end }} -dir={{ .Values.volume.dir }} \ {{- if .Values.volume.dir_idx }} @@ -170,8 +170,12 @@ spec: ports: - containerPort: {{ .Values.volume.port }} name: swfs-vol + {{- if .Values.volume.metricsPort }} + - containerPort: {{ .Values.filer.metricsPort }} + name: metrics + {{- end }} - containerPort: {{ .Values.volume.grpcPort }} - #name: swfs-vol-grpc + name: swfs-vol-grpc readinessProbe: httpGet: path: /status diff --git a/k8s/helm_charts2/values.yaml b/k8s/helm_charts2/values.yaml index 6f724879c..02978927b 100644 --- a/k8s/helm_charts2/values.yaml +++ b/k8s/helm_charts2/values.yaml @@ -39,6 +39,7 @@ master: replicas: 1 port: 9333 grpcPort: 19333 + metricsPort: 9327 ipBind: "0.0.0.0" volumePreallocate: false volumeSizeLimitMB: 1000 diff --git a/other/metrics/grafana_seaweedfs_k8s.json b/other/metrics/grafana_seaweedfs_k8s.json index 348198e52..d1b6b1714 100644 --- a/other/metrics/grafana_seaweedfs_k8s.json +++ b/other/metrics/grafana_seaweedfs_k8s.json @@ -1891,7 +1891,7 @@ "steppedLine": false, "targets": [ { - "expr": "go_memstats_alloc_bytes{namespace=~\"$namespace\", endpoint=\"swfs-filer-metrics\"}", + "expr": "go_memstats_alloc_bytes{namespace=~\"$namespace\", endpoint=\"metrics\"}", "format": "time_series", "hide": false, "interval": "", @@ -1909,7 +1909,7 @@ "refId": "A" }, { - "expr": "go_memstats_stack_inuse_bytes{namespace=~\"$namespace\", endpoint=\"swfs-filer-metrics\"}", + "expr": "go_memstats_stack_inuse_bytes{namespace=~\"$namespace\", endpoint=\"metrics\"}", "format": "time_series", "hide": true, "interval": "", @@ -1918,7 +1918,7 @@ "refId": "C" }, { - "expr": "go_memstats_heap_inuse_bytes{namespace=~\"$namespace\", endpoint=\"swfs-filer-metrics\"}", + "expr": "go_memstats_heap_inuse_bytes{namespace=~\"$namespace\", endpoint=\"metrics\"}", "format": "time_series", "hide": true, "interval": "", @@ -2019,7 +2019,7 @@ "steppedLine": false, "targets": [ { - "expr": "go_gc_duration_seconds{namespace=~\"$namespace\", endpoint=\"swfs-filer-metrics\"}", + "expr": "go_gc_duration_seconds{namespace=~\"$namespace\", endpoint=\"metrics\"}", "format": "time_series", "interval": "", "intervalFactor": 2, @@ -2119,7 +2119,7 @@ "steppedLine": false, "targets": [ { - "expr": "go_goroutines{namespace=~\"$namespace\", endpoint=\"swfs-filer-metrics\"}", + "expr": "go_goroutines{namespace=~\"$namespace\", endpoint=\"metrics\"}", "format": "time_series", "interval": "", "intervalFactor": 2, @@ -2203,7 +2203,7 @@ "value": "s3" }, "datasource": "$DS_PROMETHEUS", - "definition": "label_values({endpoint=\"swfs-filer-metrics\"}, namespace)", + "definition": "label_values({endpoint=\"metrics\"}, namespace)", "hide": 0, "includeAll": false, "label": "namespace", @@ -2216,7 +2216,7 @@ "value": "s3" } ], - "query": "label_values({endpoint=\"swfs-filer-metrics\"}, namespace)", + "query": "label_values({endpoint=\"metrics\"}, namespace)", "refresh": 0, "regex": "", "skipUrlSync": false,