diff --git a/src/lib/metric/collector.go b/src/lib/metric/collector.go index 2bf4b165d..1bc0e7213 100644 --- a/src/lib/metric/collector.go +++ b/src/lib/metric/collector.go @@ -17,14 +17,13 @@ func RegisterCollectors() { var ( // TotalInFlightGauge used to collect total in flight number - TotalInFlightGauge = prometheus.NewGaugeVec( + TotalInFlightGauge = prometheus.NewGauge( prometheus.GaugeOpts{ Namespace: os.Getenv(NamespaceEnvKey), Subsystem: os.Getenv(SubsystemEnvKey), - Name: "http_request_inflight", + Name: "http_inflight_requests", Help: "The total number of requests", }, - []string{"url"}, ) // TotalReqCnt used to collect total request counter @@ -32,10 +31,10 @@ var ( prometheus.CounterOpts{ Namespace: os.Getenv(NamespaceEnvKey), Subsystem: os.Getenv(SubsystemEnvKey), - Name: "http_request", + Name: "http_request_total", Help: "The total number of requests", }, - []string{"method", "code", "url"}, + []string{"method", "code", "operation"}, ) // TotalReqDurSummary used to collect total request duration summaries @@ -47,5 +46,5 @@ var ( Help: "The time duration of the requests", Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001}, }, - []string{"method", "url"}) + []string{"method", "operation"}) ) diff --git a/src/server/middleware/metric/metric.go b/src/server/middleware/metric/metric.go index ccdaa8f81..802b3dee2 100644 --- a/src/server/middleware/metric/metric.go +++ b/src/server/middleware/metric/metric.go @@ -1,8 +1,10 @@ package metric import ( + "context" "net/http" "strconv" + "strings" "time" "github.com/goharbor/harbor/src/core/config" @@ -10,16 +12,57 @@ import ( "github.com/goharbor/harbor/src/lib/metric" ) +// ContextOpIDKey ... +type contextOpIDKey struct{} + +const ( + // CatalogOperationID ... + CatalogOperationID = "v2_catalog" + // ListTagOperationID ... + ListTagOperationID = "v2_tags" + // ManifestOperationID ... + ManifestOperationID = "v2_manifest" + // BlobsOperationID ... + BlobsOperationID = "v2_blob" + // BlobsUploadOperationID ... + BlobsUploadOperationID = "v2_blob_upload" + // OthersOperationID ... + OthersOperationID = "v2_others" +) + +// SetMetricOpID used to set operation ID for metrics +func SetMetricOpID(ctx context.Context, value string) { + if config.Metric().Enabled { + v := ctx.Value(contextOpIDKey{}).(*string) + *v = value + } +} + +func isChartMuseumURL(url string) bool { + return strings.HasPrefix(url, "/chartrepo/") || strings.HasPrefix(url, "/api/chartrepo/") +} + func instrumentHandler(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - now, url := time.Now(), r.URL.EscapedPath() + metric.TotalInFlightGauge.Inc() + defer metric.TotalInFlightGauge.Dec() + now, rc, op := time.Now(), lib.NewResponseRecorder(w), "" + ctx := context.WithValue(r.Context(), contextOpIDKey{}, &op) + next.ServeHTTP(rc, r.WithContext(ctx)) + if len(op) == 0 && isChartMuseumURL(r.URL.Path) { + op = "chartmuseum" + } else { + // From swagger's perspective the operation of this legacy URL is unknown + op = "unknown" + } + metric.TotalReqDurSummary.WithLabelValues(r.Method, op).Observe(time.Since(now).Seconds()) + metric.TotalReqCnt.WithLabelValues(r.Method, strconv.Itoa(rc.StatusCode), op).Inc() + }) +} - metric.TotalInFlightGauge.WithLabelValues(url).Inc() - defer metric.TotalInFlightGauge.WithLabelValues(url).Dec() - rc := lib.NewResponseRecorder(w) - next.ServeHTTP(rc, r) - metric.TotalReqDurSummary.WithLabelValues(r.Method, url).Observe(time.Since(now).Seconds()) - metric.TotalReqCnt.WithLabelValues(r.Method, strconv.Itoa(rc.StatusCode), url).Inc() +func transparentHandler(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + next.ServeHTTP(w, r) }) } @@ -28,9 +71,15 @@ func Middleware() func(http.Handler) http.Handler { if config.Metric().Enabled { return instrumentHandler } + return transparentHandler +} + +// InjectOpIDMiddleware returns a middleware used for injecting operations ID +func InjectOpIDMiddleware(opID string) func(next http.Handler) http.Handler { return func(next http.Handler) http.Handler { - return http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) { - next.ServeHTTP(rw, req) + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + SetMetricOpID(r.Context(), opID) + next.ServeHTTP(w, r) }) } } diff --git a/src/server/registry/route.go b/src/server/registry/route.go index 87b24a501..7c2ae364a 100644 --- a/src/server/registry/route.go +++ b/src/server/registry/route.go @@ -20,6 +20,7 @@ import ( "github.com/goharbor/harbor/src/server/middleware/blob" "github.com/goharbor/harbor/src/server/middleware/contenttrust" "github.com/goharbor/harbor/src/server/middleware/immutable" + "github.com/goharbor/harbor/src/server/middleware/metric" "github.com/goharbor/harbor/src/server/middleware/quota" "github.com/goharbor/harbor/src/server/middleware/repoproxy" "github.com/goharbor/harbor/src/server/middleware/v2auth" @@ -36,16 +37,19 @@ func RegisterRoutes() { root.NewRoute(). Method(http.MethodGet). Path("/_catalog"). + Middleware(metric.InjectOpIDMiddleware(metric.CatalogOperationID)). Handler(newRepositoryHandler()) // list tags root.NewRoute(). Method(http.MethodGet). Path("/*/tags/list"). + Middleware(metric.InjectOpIDMiddleware(metric.ListTagOperationID)). Handler(newTagHandler()) // manifest root.NewRoute(). Method(http.MethodGet). Path("/*/manifests/:reference"). + Middleware(metric.InjectOpIDMiddleware(metric.ManifestOperationID)). Middleware(repoproxy.ManifestMiddleware()). Middleware(contenttrust.Middleware()). Middleware(vulnerable.Middleware()). @@ -53,31 +57,43 @@ func RegisterRoutes() { root.NewRoute(). Method(http.MethodHead). Path("/*/manifests/:reference"). + Middleware(metric.InjectOpIDMiddleware(metric.ManifestOperationID)). Middleware(repoproxy.ManifestMiddleware()). HandlerFunc(getManifest) root.NewRoute(). Method(http.MethodDelete). Path("/*/manifests/:reference"). + Middleware(metric.InjectOpIDMiddleware(metric.ManifestOperationID)). Middleware(quota.RefreshForProjectMiddleware()). HandlerFunc(deleteManifest) root.NewRoute(). Method(http.MethodPut). Path("/*/manifests/:reference"). + Middleware(metric.InjectOpIDMiddleware(metric.ManifestOperationID)). Middleware(repoproxy.DisableBlobAndManifestUploadMiddleware()). Middleware(immutable.Middleware()). Middleware(quota.PutManifestMiddleware()). Middleware(blob.PutManifestMiddleware()). HandlerFunc(putManifest) + // blob head + root.NewRoute(). + Method(http.MethodHead). + Path("/*/blobs/:digest"). + Middleware(metric.InjectOpIDMiddleware(metric.BlobsOperationID)). + Middleware(blob.HeadBlobMiddleware()). + Handler(proxy) // blob get root.NewRoute(). Method(http.MethodGet). Path("/*/blobs/:digest"). + Middleware(metric.InjectOpIDMiddleware(metric.BlobsOperationID)). Middleware(repoproxy.BlobGetMiddleware()). Handler(proxy) // initiate blob upload root.NewRoute(). Method(http.MethodPost). Path("/*/blobs/uploads"). + Middleware(metric.InjectOpIDMiddleware(metric.BlobsUploadOperationID)). Middleware(repoproxy.DisableBlobAndManifestUploadMiddleware()). Middleware(quota.PostInitiateBlobUploadMiddleware()). Middleware(blob.PostInitiateBlobUploadMiddleware()). @@ -86,19 +102,16 @@ func RegisterRoutes() { root.NewRoute(). Method(http.MethodPatch). Path("/*/blobs/uploads/:session_id"). + Middleware(metric.InjectOpIDMiddleware(metric.BlobsUploadOperationID)). Middleware(blob.PatchBlobUploadMiddleware()). Handler(proxy) root.NewRoute(). Method(http.MethodPut). Path("/*/blobs/uploads/:session_id"). + Middleware(metric.InjectOpIDMiddleware(metric.BlobsUploadOperationID)). Middleware(quota.PutBlobUploadMiddleware()). Middleware(blob.PutBlobUploadMiddleware()). Handler(proxy) - root.NewRoute(). - Method(http.MethodHead). - Path("/*/blobs/:digest"). - Middleware(blob.HeadBlobMiddleware()). - Handler(proxy) // others - root.NewRoute().Path("/*").Handler(proxy) + root.NewRoute().Path("/*").Middleware(metric.InjectOpIDMiddleware(metric.OthersOperationID)).Handler(proxy) } diff --git a/src/server/v2.0/handler/handler.go b/src/server/v2.0/handler/handler.go index 97004825b..7fe93465d 100644 --- a/src/server/v2.0/handler/handler.go +++ b/src/server/v2.0/handler/handler.go @@ -23,6 +23,7 @@ import ( lib_http "github.com/goharbor/harbor/src/lib/http" "github.com/goharbor/harbor/src/server/middleware" "github.com/goharbor/harbor/src/server/middleware/blob" + "github.com/goharbor/harbor/src/server/middleware/metric" "github.com/goharbor/harbor/src/server/middleware/quota" "github.com/goharbor/harbor/src/server/v2.0/restapi" ) @@ -62,6 +63,7 @@ func New() http.Handler { // function is called before the Prepare of the operation func beforePrepare(ctx context.Context, operation string, params interface{}) rmiddleware.Responder { + metric.SetMetricOpID(ctx, operation) return nil } diff --git a/tests/apitests/python/test_verify_metrics_enabled.py b/tests/apitests/python/test_verify_metrics_enabled.py index 0d4a848bf..e4bfad137 100644 --- a/tests/apitests/python/test_verify_metrics_enabled.py +++ b/tests/apitests/python/test_verify_metrics_enabled.py @@ -10,8 +10,8 @@ class TestMetricsExist(unittest.TestCase): golang_basic_metrics = ["go_gc_duration_seconds", "go_goroutines", "go_info", "go_memstats_alloc_bytes"] eigen_metrics = { - 'core': golang_basic_metrics + ["harbor_core_http_request", "harbor_core_http_request_duration_seconds", - "harbor_core_http_request_inflight"], + 'core': golang_basic_metrics + ["harbor_core_http_request_total", "harbor_core_http_request_duration_seconds", + "harbor_core_http_inflight_requests"], 'registry': golang_basic_metrics + ["registry_http_in_flight_requests"], 'exporter': golang_basic_metrics + ["harbor_image_pulled", "harbor_project_artifact_total", "harbor_project_member_total", "harbor_project_quota_byte",