diff --git a/docker/prometheus/prometheus.yml b/docker/prometheus/prometheus.yml index 34f669d56..2f0ead441 100644 --- a/docker/prometheus/prometheus.yml +++ b/docker/prometheus/prometheus.yml @@ -8,6 +8,7 @@ scrape_configs: static_configs: - targets: - 'prometheus:9090' + - 'master:9324' - 'volume:9325' - 'filer:9326' - 's3:9327' \ No newline at end of file diff --git a/docker/seaweedfs-compose.yml b/docker/seaweedfs-compose.yml index 706f10180..cce1c39ef 100644 --- a/docker/seaweedfs-compose.yml +++ b/docker/seaweedfs-compose.yml @@ -6,7 +6,8 @@ services: ports: - 9333:9333 - 19333:19333 - command: "master -ip=master -ip.bind=0.0.0.0" + - 9324:9324 + command: "master -ip=master -ip.bind=0.0.0.0 -metricsPort=9324" volume: image: chrislusf/seaweedfs # use a remote image ports: diff --git a/weed/stats/metrics.go b/weed/stats/metrics.go index 8381999c2..943aafff9 100644 --- a/weed/stats/metrics.go +++ b/weed/stats/metrics.go @@ -44,6 +44,14 @@ var ( Help: "Counter of master received heartbeat.", }, []string{"type"}) + MasterReplicaPlacementMismatch = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: "SeaweedFS", + Subsystem: "master", + Name: "replica_placement_mismatch", + Help: "replica placement mismatch", + }, []string{"collection", "id"}) + MasterLeaderChangeCounter = prometheus.NewCounterVec( prometheus.CounterOpts{ Namespace: "SeaweedFS", @@ -165,6 +173,7 @@ func init() { Gather.MustRegister(MasterRaftIsleader) Gather.MustRegister(MasterReceivedHeartbeatCounter) Gather.MustRegister(MasterLeaderChangeCounter) + Gather.MustRegister(MasterReplicaPlacementMismatch) Gather.MustRegister(FilerRequestCounter) Gather.MustRegister(FilerRequestHistogram) diff --git a/weed/topology/node.go b/weed/topology/node.go index 4772cb411..c5956177a 100644 --- a/weed/topology/node.go +++ b/weed/topology/node.go @@ -3,6 +3,7 @@ package topology import ( "errors" "github.com/chrislusf/seaweedfs/weed/glog" + "github.com/chrislusf/seaweedfs/weed/stats" "github.com/chrislusf/seaweedfs/weed/storage/erasure_coding" "github.com/chrislusf/seaweedfs/weed/storage/needle" "github.com/chrislusf/seaweedfs/weed/storage/types" @@ -246,6 +247,14 @@ func (n *NodeImpl) CollectDeadNodeAndFullVolumes(freshThreshHold int64, volumeSi } else if float64(v.Size) > float64(volumeSizeLimit)*growThreshold { n.GetTopology().chanCrowdedVolumes <- v } + copyCount := v.ReplicaPlacement.GetCopyCount() + if copyCount > 1 { + if copyCount > len(n.GetTopology().Lookup(v.Collection, v.Id)) { + stats.MasterReplicaPlacementMismatch.WithLabelValues(v.Collection, v.Id.String()).Set(1) + } else { + stats.MasterReplicaPlacementMismatch.WithLabelValues(v.Collection, v.Id.String()).Set(0) + } + } } } } else {