refactor FilerRequest metrics (#3402)

* refactor FilerRequest metrics

* avoid double count proxy

* defer to

weed/operation/upload_content.go

@@ -7,6 +7,7 @@ import (
 	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
 	"github.com/seaweedfs/seaweedfs/weed/security"
+	"github.com/seaweedfs/seaweedfs/weed/stats"
 	"github.com/seaweedfs/seaweedfs/weed/util"
 	"io"
 	"mime"
@@ -267,6 +268,7 @@ func upload_content(fillBufferFunction func(w io.Writer) error, originalDataSize
 		if strings.Contains(post_err.Error(), "connection reset by peer") ||
 			strings.Contains(post_err.Error(), "use of closed network connection") {
 			glog.V(1).Infof("repeat error upload request %s: %v", option.UploadUrl, postErr)
+			stats.FilerRequestCounter.WithLabelValues(stats.RepeatErrorUploadContent).Inc()
 			resp, post_err = HttpClient.Do(req)
 		}
 	}

weed/server/filer_server_handlers.go

@@ -14,13 +14,9 @@ import (
 )
 
 func (fs *FilerServer) filerHandler(w http.ResponseWriter, r *http.Request) {
-
 	start := time.Now()
-
 	if r.Method == "OPTIONS" {
-		stats.FilerRequestCounter.WithLabelValues("options").Inc()
 		OptionsHandler(w, r, false)
-		stats.FilerRequestHistogram.WithLabelValues("options").Observe(time.Since(start).Seconds())
 		return
 	}
 
@@ -36,12 +32,17 @@ func (fs *FilerServer) filerHandler(w http.ResponseWriter, r *http.Request) {
 		fileId = r.RequestURI[len("/?proxyChunkId="):]
 	}
 	if fileId != "" {
-		stats.FilerRequestCounter.WithLabelValues("proxy").Inc()
+		stats.FilerRequestCounter.WithLabelValues(stats.ChunkProxy).Inc()
 		fs.proxyToVolumeServer(w, r, fileId)
-		stats.FilerRequestHistogram.WithLabelValues("proxy").Observe(time.Since(start).Seconds())
+		stats.FilerRequestHistogram.WithLabelValues(stats.ChunkProxy).Observe(time.Since(start).Seconds())
 		return
 	}
 
+	stats.FilerRequestCounter.WithLabelValues(r.Method).Inc()
+	defer func() {
+		stats.FilerRequestHistogram.WithLabelValues(r.Method).Observe(time.Since(start).Seconds())
+	}()
+
 	w.Header().Set("Server", "SeaweedFS Filer "+util.VERSION)
 	if r.Header.Get("Origin") != "" {
 		w.Header().Set("Access-Control-Allow-Origin", "*")
@@ -49,23 +50,16 @@ func (fs *FilerServer) filerHandler(w http.ResponseWriter, r *http.Request) {
 	}
 	switch r.Method {
 	case "GET":
-		stats.FilerRequestCounter.WithLabelValues("get").Inc()
 		fs.GetOrHeadHandler(w, r)
-		stats.FilerRequestHistogram.WithLabelValues("get").Observe(time.Since(start).Seconds())
 	case "HEAD":
-		stats.FilerRequestCounter.WithLabelValues("head").Inc()
 		fs.GetOrHeadHandler(w, r)
-		stats.FilerRequestHistogram.WithLabelValues("head").Observe(time.Since(start).Seconds())
 	case "DELETE":
-		stats.FilerRequestCounter.WithLabelValues("delete").Inc()
 		if _, ok := r.URL.Query()["tagging"]; ok {
 			fs.DeleteTaggingHandler(w, r)
 		} else {
 			fs.DeleteHandler(w, r)
 		}
-		stats.FilerRequestHistogram.WithLabelValues("delete").Observe(time.Since(start).Seconds())
 	case "POST", "PUT":
-
 		// wait until in flight data is less than the limit
 		contentLength := getContentLength(r)
 		fs.inFlightDataLimitCond.L.Lock()
@@ -81,17 +75,13 @@ func (fs *FilerServer) filerHandler(w http.ResponseWriter, r *http.Request) {
 		}()
 
 		if r.Method == "PUT" {
-			stats.FilerRequestCounter.WithLabelValues("put").Inc()
 			if _, ok := r.URL.Query()["tagging"]; ok {
 				fs.PutTaggingHandler(w, r)
 			} else {
 				fs.PostHandler(w, r, contentLength)
 			}
-			stats.FilerRequestHistogram.WithLabelValues("put").Observe(time.Since(start).Seconds())
 		} else { // method == "POST"
-			stats.FilerRequestCounter.WithLabelValues("post").Inc()
 			fs.PostHandler(w, r, contentLength)
-			stats.FilerRequestHistogram.WithLabelValues("post").Observe(time.Since(start).Seconds())
 		}
 	}
 }
@@ -99,12 +89,13 @@ func (fs *FilerServer) filerHandler(w http.ResponseWriter, r *http.Request) {
 func (fs *FilerServer) readonlyFilerHandler(w http.ResponseWriter, r *http.Request) {
 
 	start := time.Now()
-
+	stats.FilerRequestCounter.WithLabelValues(r.Method).Inc()
+	defer func() {
+		stats.FilerRequestHistogram.WithLabelValues(r.Method).Observe(time.Since(start).Seconds())
+	}()
 	// We handle OPTIONS first because it never should be authenticated
 	if r.Method == "OPTIONS" {
-		stats.FilerRequestCounter.WithLabelValues("options").Inc()
 		OptionsHandler(w, r, true)
-		stats.FilerRequestHistogram.WithLabelValues("options").Observe(time.Since(start).Seconds())
 		return
 	}
 
@@ -120,13 +111,9 @@ func (fs *FilerServer) readonlyFilerHandler(w http.ResponseWriter, r *http.Reque
 	}
 	switch r.Method {
 	case "GET":
-		stats.FilerRequestCounter.WithLabelValues("get").Inc()
 		fs.GetOrHeadHandler(w, r)
-		stats.FilerRequestHistogram.WithLabelValues("get").Observe(time.Since(start).Seconds())
 	case "HEAD":
-		stats.FilerRequestCounter.WithLabelValues("head").Inc()
 		fs.GetOrHeadHandler(w, r)
-		stats.FilerRequestHistogram.WithLabelValues("head").Observe(time.Since(start).Seconds())
 	}
 }
 

weed/server/filer_server_handlers_read_dir.go

@@ -18,7 +18,7 @@ import (
 // is empty.
 func (fs *FilerServer) listDirectoryHandler(w http.ResponseWriter, r *http.Request) {
 
-	stats.FilerRequestCounter.WithLabelValues("list").Inc()
+	stats.FilerRequestCounter.WithLabelValues(stats.DirList).Inc()
 
 	path := r.URL.Path
 	if strings.HasSuffix(path, "/") && len(path) > 1 {

weed/server/filer_server_handlers_write.go

@@ -34,9 +34,11 @@ type FilerPostResult struct {
 
 func (fs *FilerServer) assignNewFileInfo(so *operation.StorageOption) (fileId, urlLocation string, auth security.EncodedJwt, err error) {
 
-	stats.FilerRequestCounter.WithLabelValues("assign").Inc()
+	stats.FilerRequestCounter.WithLabelValues(stats.ChunkAssign).Inc()
 	start := time.Now()
-	defer func() { stats.FilerRequestHistogram.WithLabelValues("assign").Observe(time.Since(start).Seconds()) }()
+	defer func() {
+		stats.FilerRequestHistogram.WithLabelValues(stats.ChunkAssign).Observe(time.Since(start).Seconds())
+	}()
 
 	ar, altRequest := so.ToAssignRequests(1)
 

weed/server/filer_server_handlers_write_autochunk.go

@@ -16,7 +16,6 @@ import (
 	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/operation"
 	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
-	"github.com/seaweedfs/seaweedfs/weed/stats"
 	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
 	"github.com/seaweedfs/seaweedfs/weed/util"
 )
@@ -34,12 +33,6 @@ func (fs *FilerServer) autoChunk(ctx context.Context, w http.ResponseWriter, r *
 
 	chunkSize := 1024 * 1024 * maxMB
 
-	stats.FilerRequestCounter.WithLabelValues("chunk").Inc()
-	start := time.Now()
-	defer func() {
-		stats.FilerRequestHistogram.WithLabelValues("chunk").Observe(time.Since(start).Seconds())
-	}()
-
 	var reply *FilerPostResult
 	var err error
 	var md5bytes []byte

weed/server/filer_server_handlers_write_upload.go

@@ -90,8 +90,11 @@ func (fs *FilerServer) uploadReaderToChunks(w http.ResponseWriter, r *http.Reque
 				bufPool.Put(bytesBuffer)
 				atomic.AddInt64(&bytesBufferCounter, -1)
 				bytesBufferLimitCond.Signal()
+				stats.FilerRequestCounter.WithLabelValues(stats.ContentSaveToFiler).Inc()
 				break
 			}
+		} else {
+			stats.FilerRequestCounter.WithLabelValues(stats.AutoChunk).Inc()
 		}
 
 		wg.Add(1)
@@ -138,10 +141,10 @@ func (fs *FilerServer) uploadReaderToChunks(w http.ResponseWriter, r *http.Reque
 
 func (fs *FilerServer) doUpload(urlLocation string, limitedReader io.Reader, fileName string, contentType string, pairMap map[string]string, auth security.EncodedJwt) (*operation.UploadResult, error, []byte) {
 
-	stats.FilerRequestCounter.WithLabelValues("chunkUpload").Inc()
+	stats.FilerRequestCounter.WithLabelValues(stats.ChunkUpload).Inc()
 	start := time.Now()
 	defer func() {
-		stats.FilerRequestHistogram.WithLabelValues("chunkUpload").Observe(time.Since(start).Seconds())
+		stats.FilerRequestHistogram.WithLabelValues(stats.ChunkUpload).Observe(time.Since(start).Seconds())
 	}()
 
 	uploadOption := &operation.UploadOption{
@@ -155,7 +158,7 @@ func (fs *FilerServer) doUpload(urlLocation string, limitedReader io.Reader, fil
 	}
 	uploadResult, err, data := operation.Upload(limitedReader, uploadOption)
 	if uploadResult != nil && uploadResult.RetryCount > 0 {
-		stats.FilerRequestCounter.WithLabelValues("chunkUploadRetry").Add(float64(uploadResult.RetryCount))
+		stats.FilerRequestCounter.WithLabelValues(stats.ChunkUploadRetry).Add(float64(uploadResult.RetryCount))
 	}
 	return uploadResult, err, data
 }
@@ -173,6 +176,7 @@ func (fs *FilerServer) dataToChunk(fileName, contentType string, data []byte, ch
 		fileId, urlLocation, auth, uploadErr = fs.assignNewFileInfo(so)
 		if uploadErr != nil {
 			glog.V(4).Infof("retry later due to assign error: %v", uploadErr)
+			stats.FilerRequestCounter.WithLabelValues(stats.ChunkAssignRetry).Inc()
 			time.Sleep(time.Duration(i+1) * 251 * time.Millisecond)
 			continue
 		}
@@ -181,6 +185,7 @@ func (fs *FilerServer) dataToChunk(fileName, contentType string, data []byte, ch
 		uploadResult, uploadErr, _ = fs.doUpload(urlLocation, dataReader, fileName, contentType, nil, auth)
 		if uploadErr != nil {
 			glog.V(4).Infof("retry later due to upload error: %v", uploadErr)
+			stats.FilerRequestCounter.WithLabelValues(stats.ChunkDoUploadRetry).Inc()
 			time.Sleep(time.Duration(i+1) * 251 * time.Millisecond)
 			continue
 		}

weed/stats/metrics_names.go

@@ -20,14 +20,24 @@ const (
 	FailedToKeepConnected = "failedToKeepConnected"
 	FailedToSend          = "failedToSend"
 	FailedToReceive       = "failedToReceive"
-	RedirectedToleader    = "redirectedToleader"
+	RedirectedToLeader    = "redirectedToLeader"
 	OnPeerUpdate          = "onPeerUpdate"
 	Failed                = "failed"
 
 	// filer handler
-	ErrorReadNotFound = "read.notfound"
+	DirList                  = "dirList"
-	ErrorReadInternal = "read.internalerror"
+	ContentSaveToFiler       = "contentSaveToFiler"
-	ErrorWriteEntry   = "write.entry.failed"
+	AutoChunk                = "autoChunk"
-	ErrorReadCache    = "read.cache.failed"
+	ChunkProxy               = "chunkProxy"
-	ErrorReadStream   = "read.stream.failed"
+	ChunkAssign              = "chunkAssign"
+	ChunkUpload              = "chunkUpload"
+	ChunkDoUploadRetry       = "chunkDoUploadRetry"
+	ChunkUploadRetry         = "chunkUploadRetry"
+	ChunkAssignRetry         = "chunkAssignRetry"
+	ErrorReadNotFound        = "read.notfound"
+	ErrorReadInternal        = "read.internal.error"
+	ErrorWriteEntry          = "write.entry.failed"
+	RepeatErrorUploadContent = "upload.content.repeat.failed"
+	ErrorReadCache           = "read.cache.failed"
+	ErrorReadStream          = "read.stream.failed"
 )

weed/wdclient/masterclient.go

@@ -178,7 +178,7 @@ func (mc *MasterClient) tryConnectToMaster(master pb.ServerAddress) (nextHintedL
 			if resp.VolumeLocation.Leader != "" && string(master) != resp.VolumeLocation.Leader {
 				glog.V(0).Infof("master %v redirected to leader %v", master, resp.VolumeLocation.Leader)
 				nextHintedLeader = pb.ServerAddress(resp.VolumeLocation.Leader)
-				stats.MasterClientConnectCounter.WithLabelValues(stats.RedirectedToleader).Inc()
+				stats.MasterClientConnectCounter.WithLabelValues(stats.RedirectedToLeader).Inc()
 				return nil
 			}
 			//mc.vidMap = newVidMap("")
@@ -203,7 +203,7 @@ func (mc *MasterClient) tryConnectToMaster(master pb.ServerAddress) (nextHintedL
 				if resp.VolumeLocation.Leader != "" && string(mc.currentMaster) != resp.VolumeLocation.Leader {
 					glog.V(0).Infof("currentMaster %v redirected to leader %v", mc.currentMaster, resp.VolumeLocation.Leader)
 					nextHintedLeader = pb.ServerAddress(resp.VolumeLocation.Leader)
-					stats.MasterClientConnectCounter.WithLabelValues(stats.RedirectedToleader).Inc()
+					stats.MasterClientConnectCounter.WithLabelValues(stats.RedirectedToLeader).Inc()
 					return nil
 				}