// Copyright (c) 2024 Mattermost Community Enterprise // Metrics interface method implementations package metrics import ( "fmt" "github.com/mattermost/logr/v2" "github.com/mattermost/mattermost/server/public/model" "github.com/mattermost/mattermost/server/public/shared/mlog" "github.com/prometheus/client_golang/prometheus" ) // Post metrics func (m *MetricsImpl) IncrementPostCreate() { m.postCreate.Inc() } func (m *MetricsImpl) IncrementWebhookPost() { m.webhookPost.Inc() } func (m *MetricsImpl) IncrementPostSentEmail() { m.postSentEmail.Inc() } func (m *MetricsImpl) IncrementPostSentPush() { m.postSentPush.Inc() } func (m *MetricsImpl) IncrementPostBroadcast() { m.postBroadcast.Inc() } func (m *MetricsImpl) IncrementPostFileAttachment(count int) { m.postFileAttachment.Add(float64(count)) } // HTTP metrics func (m *MetricsImpl) IncrementHTTPRequest() { m.httpRequest.Inc() } func (m *MetricsImpl) IncrementHTTPError() { m.httpError.Inc() } // Cluster metrics func (m *MetricsImpl) IncrementClusterRequest() { m.clusterRequest.Inc() } func (m *MetricsImpl) ObserveClusterRequestDuration(elapsed float64) { m.clusterRequestTime.Observe(elapsed) } func (m *MetricsImpl) IncrementClusterEventType(eventType model.ClusterEvent) { m.clusterEventCounter.WithLabelValues(string(eventType)).Inc() } // Login metrics func (m *MetricsImpl) IncrementLogin() { m.login.Inc() } func (m *MetricsImpl) IncrementLoginFail() { m.loginFail.Inc() } // ETag cache metrics func (m *MetricsImpl) IncrementEtagHitCounter(route string) { m.etagHit.WithLabelValues(route).Inc() } func (m *MetricsImpl) IncrementEtagMissCounter(route string) { m.etagMiss.WithLabelValues(route).Inc() } // Memory cache metrics func (m *MetricsImpl) IncrementMemCacheHitCounter(cacheName string) { m.memCacheHit.WithLabelValues(cacheName).Inc() } func (m *MetricsImpl) IncrementMemCacheMissCounter(cacheName string) { m.memCacheMiss.WithLabelValues(cacheName).Inc() } func (m *MetricsImpl) IncrementMemCacheInvalidationCounter(cacheName string) { m.memCacheInvalidation.WithLabelValues(cacheName).Inc() } func (m *MetricsImpl) IncrementMemCacheMissCounterSession() { m.sessionCacheMiss.Inc() } func (m *MetricsImpl) IncrementMemCacheHitCounterSession() { m.sessionCacheHit.Inc() } func (m *MetricsImpl) IncrementMemCacheInvalidationCounterSession() { m.sessionCacheInvalidation.Inc() } func (m *MetricsImpl) AddMemCacheHitCounter(cacheName string, amount float64) { m.memCacheHit.WithLabelValues(cacheName).Add(amount) } func (m *MetricsImpl) AddMemCacheMissCounter(cacheName string, amount float64) { m.memCacheMiss.WithLabelValues(cacheName).Add(amount) } // WebSocket metrics func (m *MetricsImpl) IncrementWebsocketEvent(eventType model.WebsocketEventType) { m.websocketEvent.WithLabelValues(string(eventType)).Inc() } func (m *MetricsImpl) IncrementWebSocketBroadcast(eventType model.WebsocketEventType) { m.websocketBroadcast.WithLabelValues(string(eventType)).Inc() } func (m *MetricsImpl) IncrementWebSocketBroadcastBufferSize(hub string, amount float64) { m.websocketBroadcastBuffer.WithLabelValues(hub).Add(amount) } func (m *MetricsImpl) DecrementWebSocketBroadcastBufferSize(hub string, amount float64) { m.websocketBroadcastBuffer.WithLabelValues(hub).Sub(amount) } func (m *MetricsImpl) IncrementWebSocketBroadcastUsersRegistered(hub string, amount float64) { m.websocketBroadcastUsers.WithLabelValues(hub).Add(amount) } func (m *MetricsImpl) DecrementWebSocketBroadcastUsersRegistered(hub string, amount float64) { m.websocketBroadcastUsers.WithLabelValues(hub).Sub(amount) } func (m *MetricsImpl) IncrementWebsocketReconnectEventWithDisconnectErrCode(eventType string, disconnectErrCode string) { m.websocketReconnect.WithLabelValues(eventType, disconnectErrCode).Inc() } func (m *MetricsImpl) IncrementHTTPWebSockets(originClient string) { m.httpWebsockets.WithLabelValues(originClient).Inc() } func (m *MetricsImpl) DecrementHTTPWebSockets(originClient string) { m.httpWebsockets.WithLabelValues(originClient).Dec() } // Search metrics func (m *MetricsImpl) IncrementPostsSearchCounter() { m.postsSearch.Inc() } func (m *MetricsImpl) ObservePostsSearchDuration(elapsed float64) { m.postsSearchTime.Observe(elapsed) } func (m *MetricsImpl) IncrementFilesSearchCounter() { m.filesSearch.Inc() } func (m *MetricsImpl) ObserveFilesSearchDuration(elapsed float64) { m.filesSearchTime.Observe(elapsed) } func (m *MetricsImpl) ObserveStoreMethodDuration(method, success string, elapsed float64) { m.storeMethodTime.WithLabelValues(method, success).Observe(elapsed) } func (m *MetricsImpl) ObserveAPIEndpointDuration(endpoint, method, statusCode, originClient, pageLoadContext string, elapsed float64) { m.apiEndpointTime.WithLabelValues(endpoint, method, statusCode, originClient, pageLoadContext).Observe(elapsed) } func (m *MetricsImpl) ObserveRedisEndpointDuration(cacheName, operation string, elapsed float64) { m.redisEndpointTime.WithLabelValues(cacheName, operation).Observe(elapsed) } // Index metrics func (m *MetricsImpl) IncrementPostIndexCounter() { m.postIndex.Inc() } func (m *MetricsImpl) IncrementFileIndexCounter() { m.fileIndex.Inc() } func (m *MetricsImpl) IncrementUserIndexCounter() { m.userIndex.Inc() } func (m *MetricsImpl) IncrementChannelIndexCounter() { m.channelIndex.Inc() } // Plugin metrics func (m *MetricsImpl) ObservePluginHookDuration(pluginID, hookName string, success bool, elapsed float64) { m.pluginHookTime.WithLabelValues(pluginID, hookName, fmt.Sprintf("%t", success)).Observe(elapsed) } func (m *MetricsImpl) ObservePluginMultiHookIterationDuration(pluginID string, elapsed float64) { m.pluginMultiHookIterTime.WithLabelValues(pluginID).Observe(elapsed) } func (m *MetricsImpl) ObservePluginMultiHookDuration(elapsed float64) { m.pluginMultiHookTime.Observe(elapsed) } func (m *MetricsImpl) ObservePluginAPIDuration(pluginID, apiName string, success bool, elapsed float64) { m.pluginAPITime.WithLabelValues(pluginID, apiName, fmt.Sprintf("%t", success)).Observe(elapsed) } // Enabled users func (m *MetricsImpl) ObserveEnabledUsers(users int64) { m.enabledUsers.Set(float64(users)) } // Logger metrics collector func (m *MetricsImpl) GetLoggerMetricsCollector() mlog.MetricsCollector { return &LoggerMetricsCollector{metrics: m} } // Remote cluster metrics func (m *MetricsImpl) IncrementRemoteClusterMsgSentCounter(remoteID string) { m.remoteClusterMsgSent.WithLabelValues(remoteID).Inc() } func (m *MetricsImpl) IncrementRemoteClusterMsgReceivedCounter(remoteID string) { m.remoteClusterMsgReceived.WithLabelValues(remoteID).Inc() } func (m *MetricsImpl) IncrementRemoteClusterMsgErrorsCounter(remoteID string, timeout bool) { m.remoteClusterMsgErrors.WithLabelValues(remoteID, fmt.Sprintf("%t", timeout)).Inc() } func (m *MetricsImpl) ObserveRemoteClusterPingDuration(remoteID string, elapsed float64) { m.remoteClusterPingTime.WithLabelValues(remoteID).Observe(elapsed) } func (m *MetricsImpl) ObserveRemoteClusterClockSkew(remoteID string, skew float64) { m.remoteClusterClockSkew.WithLabelValues(remoteID).Set(skew) } func (m *MetricsImpl) IncrementRemoteClusterConnStateChangeCounter(remoteID string, online bool) { m.remoteClusterConnState.WithLabelValues(remoteID, fmt.Sprintf("%t", online)).Inc() } // Shared channels metrics func (m *MetricsImpl) IncrementSharedChannelsSyncCounter(remoteID string) { m.sharedChannelsSync.WithLabelValues(remoteID).Inc() } func (m *MetricsImpl) ObserveSharedChannelsTaskInQueueDuration(elapsed float64) { m.sharedChannelsTaskQueueTime.Observe(elapsed) } func (m *MetricsImpl) ObserveSharedChannelsQueueSize(size int64) { m.sharedChannelsQueueSize.Set(float64(size)) } func (m *MetricsImpl) ObserveSharedChannelsSyncCollectionDuration(remoteID string, elapsed float64) { m.sharedChannelsSyncCollectionTime.WithLabelValues(remoteID).Observe(elapsed) } func (m *MetricsImpl) ObserveSharedChannelsSyncSendDuration(remoteID string, elapsed float64) { m.sharedChannelsSyncSendTime.WithLabelValues(remoteID).Observe(elapsed) } func (m *MetricsImpl) ObserveSharedChannelsSyncCollectionStepDuration(remoteID string, step string, elapsed float64) { m.sharedChannelsSyncCollectionStep.WithLabelValues(remoteID, step).Observe(elapsed) } func (m *MetricsImpl) ObserveSharedChannelsSyncSendStepDuration(remoteID string, step string, elapsed float64) { m.sharedChannelsSyncSendStep.WithLabelValues(remoteID, step).Observe(elapsed) } // Job metrics func (m *MetricsImpl) IncrementJobActive(jobType string) { m.jobActive.WithLabelValues(jobType).Inc() } func (m *MetricsImpl) DecrementJobActive(jobType string) { m.jobActive.WithLabelValues(jobType).Dec() } // Replica lag metrics func (m *MetricsImpl) SetReplicaLagAbsolute(node string, value float64) { m.replicaLagAbsolute.WithLabelValues(node).Set(value) } func (m *MetricsImpl) SetReplicaLagTime(node string, value float64) { m.replicaLagTime.WithLabelValues(node).Set(value) } // Notification metrics func (m *MetricsImpl) IncrementNotificationCounter(notificationType model.NotificationType, platform string) { m.notificationCounter.WithLabelValues(string(notificationType), platform).Inc() } func (m *MetricsImpl) IncrementNotificationAckCounter(notificationType model.NotificationType, platform string) { m.notificationAck.WithLabelValues(string(notificationType), platform).Inc() } func (m *MetricsImpl) IncrementNotificationSuccessCounter(notificationType model.NotificationType, platform string) { m.notificationSuccess.WithLabelValues(string(notificationType), platform).Inc() } func (m *MetricsImpl) IncrementNotificationErrorCounter(notificationType model.NotificationType, errorReason model.NotificationReason, platform string) { m.notificationError.WithLabelValues(string(notificationType), string(errorReason), platform).Inc() } func (m *MetricsImpl) IncrementNotificationNotSentCounter(notificationType model.NotificationType, notSentReason model.NotificationReason, platform string) { m.notificationNotSent.WithLabelValues(string(notificationType), string(notSentReason), platform).Inc() } func (m *MetricsImpl) IncrementNotificationUnsupportedCounter(notificationType model.NotificationType, notSentReason model.NotificationReason, platform string) { m.notificationUnsupported.WithLabelValues(string(notificationType), string(notSentReason), platform).Inc() } // Client performance metrics func (m *MetricsImpl) ObserveClientTimeToFirstByte(platform, agent, userID string, elapsed float64) { m.clientTimeToFirstByte.WithLabelValues(platform, agent, userID).Observe(elapsed) } func (m *MetricsImpl) ObserveClientTimeToLastByte(platform, agent, userID string, elapsed float64) { m.clientTimeToLastByte.WithLabelValues(platform, agent, userID).Observe(elapsed) } func (m *MetricsImpl) ObserveClientTimeToDomInteractive(platform, agent, userID string, elapsed float64) { m.clientTimeToDomInteractive.WithLabelValues(platform, agent, userID).Observe(elapsed) } func (m *MetricsImpl) ObserveClientSplashScreenEnd(platform, agent, pageType, userID string, elapsed float64) { m.clientSplashScreenEnd.WithLabelValues(platform, agent, pageType, userID).Observe(elapsed) } func (m *MetricsImpl) ObserveClientFirstContentfulPaint(platform, agent, userID string, elapsed float64) { m.clientFirstContentfulPaint.WithLabelValues(platform, agent, userID).Observe(elapsed) } func (m *MetricsImpl) ObserveClientLargestContentfulPaint(platform, agent, region, userID string, elapsed float64) { m.clientLargestContentfulPaint.WithLabelValues(platform, agent, region, userID).Observe(elapsed) } func (m *MetricsImpl) ObserveClientInteractionToNextPaint(platform, agent, interaction, userID string, elapsed float64) { m.clientInteractionToNextPaint.WithLabelValues(platform, agent, interaction, userID).Observe(elapsed) } func (m *MetricsImpl) ObserveClientCumulativeLayoutShift(platform, agent, userID string, elapsed float64) { m.clientCumulativeLayoutShift.WithLabelValues(platform, agent, userID).Observe(elapsed) } func (m *MetricsImpl) IncrementClientLongTasks(platform, agent, userID string, inc float64) { m.clientLongTasks.WithLabelValues(platform, agent, userID).Add(inc) } func (m *MetricsImpl) ObserveClientPageLoadDuration(platform, agent, userID string, elapsed float64) { m.clientPageLoadDuration.WithLabelValues(platform, agent, userID).Observe(elapsed) } func (m *MetricsImpl) ObserveClientChannelSwitchDuration(platform, agent, fresh, userID string, elapsed float64) { m.clientChannelSwitchDuration.WithLabelValues(platform, agent, fresh, userID).Observe(elapsed) } func (m *MetricsImpl) ObserveClientTeamSwitchDuration(platform, agent, fresh, userID string, elapsed float64) { m.clientTeamSwitchDuration.WithLabelValues(platform, agent, fresh, userID).Observe(elapsed) } func (m *MetricsImpl) ObserveClientRHSLoadDuration(platform, agent, userID string, elapsed float64) { m.clientRHSLoadDuration.WithLabelValues(platform, agent, userID).Observe(elapsed) } func (m *MetricsImpl) ObserveGlobalThreadsLoadDuration(platform, agent, userID string, elapsed float64) { m.globalThreadsLoadDuration.WithLabelValues(platform, agent, userID).Observe(elapsed) } // Mobile client metrics func (m *MetricsImpl) ObserveMobileClientLoadDuration(platform string, elapsed float64) { m.mobileClientLoadDuration.WithLabelValues(platform).Observe(elapsed) } func (m *MetricsImpl) ObserveMobileClientChannelSwitchDuration(platform string, elapsed float64) { m.mobileClientChannelSwitchDuration.WithLabelValues(platform).Observe(elapsed) } func (m *MetricsImpl) ObserveMobileClientTeamSwitchDuration(platform string, elapsed float64) { m.mobileClientTeamSwitchDuration.WithLabelValues(platform).Observe(elapsed) } func (m *MetricsImpl) ObserveMobileClientNetworkRequestsAverageSpeed(platform, agent, networkRequestGroup string, speed float64) { m.mobileClientNetworkMetrics.WithLabelValues(platform, agent, networkRequestGroup, "average_speed").Observe(speed) } func (m *MetricsImpl) ObserveMobileClientNetworkRequestsEffectiveLatency(platform, agent, networkRequestGroup string, latency float64) { m.mobileClientNetworkMetrics.WithLabelValues(platform, agent, networkRequestGroup, "effective_latency").Observe(latency) } func (m *MetricsImpl) ObserveMobileClientNetworkRequestsElapsedTime(platform, agent, networkRequestGroup string, elapsedTime float64) { m.mobileClientNetworkMetrics.WithLabelValues(platform, agent, networkRequestGroup, "elapsed_time").Observe(elapsedTime) } func (m *MetricsImpl) ObserveMobileClientNetworkRequestsLatency(platform, agent, networkRequestGroup string, latency float64) { m.mobileClientNetworkMetrics.WithLabelValues(platform, agent, networkRequestGroup, "latency").Observe(latency) } func (m *MetricsImpl) ObserveMobileClientNetworkRequestsTotalCompressedSize(platform, agent, networkRequestGroup string, size float64) { m.mobileClientNetworkMetrics.WithLabelValues(platform, agent, networkRequestGroup, "total_compressed_size").Observe(size) } func (m *MetricsImpl) ObserveMobileClientNetworkRequestsTotalParallelRequests(platform, agent, networkRequestGroup string, count float64) { m.mobileClientNetworkMetrics.WithLabelValues(platform, agent, networkRequestGroup, "total_parallel_requests").Observe(count) } func (m *MetricsImpl) ObserveMobileClientNetworkRequestsTotalRequests(platform, agent, networkRequestGroup string, count float64) { m.mobileClientNetworkMetrics.WithLabelValues(platform, agent, networkRequestGroup, "total_requests").Observe(count) } func (m *MetricsImpl) ObserveMobileClientNetworkRequestsTotalSequentialRequests(platform, agent, networkRequestGroup string, count float64) { m.mobileClientNetworkMetrics.WithLabelValues(platform, agent, networkRequestGroup, "total_sequential_requests").Observe(count) } func (m *MetricsImpl) ObserveMobileClientNetworkRequestsTotalSize(platform, agent, networkRequestGroup string, size float64) { m.mobileClientNetworkMetrics.WithLabelValues(platform, agent, networkRequestGroup, "total_size").Observe(size) } func (m *MetricsImpl) ClearMobileClientSessionMetadata() { m.mobileClientSessionMetadata.Reset() } func (m *MetricsImpl) ObserveMobileClientSessionMetadata(version string, platform string, value float64, notificationDisabled string) { m.mobileClientSessionMetadata.WithLabelValues(version, platform, notificationDisabled).Set(value) } // Desktop metrics func (m *MetricsImpl) ObserveDesktopCpuUsage(platform, version, process string, usage float64) { m.desktopCpuUsage.WithLabelValues(platform, version, process).Set(usage) } func (m *MetricsImpl) ObserveDesktopMemoryUsage(platform, version, process string, usage float64) { m.desktopMemoryUsage.WithLabelValues(platform, version, process).Set(usage) } // Access control metrics func (m *MetricsImpl) ObserveAccessControlSearchQueryDuration(value float64) { m.accessControlSearchQuery.Observe(value) } func (m *MetricsImpl) ObserveAccessControlExpressionCompileDuration(value float64) { m.accessControlExpressionCompile.Observe(value) } func (m *MetricsImpl) ObserveAccessControlEvaluateDuration(value float64) { m.accessControlEvaluate.Observe(value) } func (m *MetricsImpl) IncrementAccessControlCacheInvalidation() { m.accessControlCacheInvalidation.Inc() } // LoggerMetricsCollector implements mlog.MetricsCollector (logr.MetricsCollector) type LoggerMetricsCollector struct { metrics *MetricsImpl } // simpleCounter implements logr.Counter type simpleCounter struct { counter prometheus.Counter } func (c *simpleCounter) Inc() { if c.counter != nil { c.counter.Inc() } } func (c *simpleCounter) Add(v float64) { if c.counter != nil { c.counter.Add(v) } } // simpleGauge implements logr.Gauge type simpleGauge struct { gauge prometheus.Gauge } func (g *simpleGauge) Set(v float64) { if g.gauge != nil { g.gauge.Set(v) } } func (g *simpleGauge) Add(v float64) { if g.gauge != nil { g.gauge.Add(v) } } func (g *simpleGauge) Sub(v float64) { if g.gauge != nil { g.gauge.Sub(v) } } // QueueSizeGauge returns a Gauge for tracking queue size func (c *LoggerMetricsCollector) QueueSizeGauge(target string) (logr.Gauge, error) { gauge := prometheus.NewGauge(prometheus.GaugeOpts{ Namespace: MetricsNamespace, Subsystem: "logging", Name: "queue_size", Help: "Logging queue size", ConstLabels: prometheus.Labels{"target": target}, }) // Try to register, ignore if already registered _ = c.metrics.registry.Register(gauge) return &simpleGauge{gauge: gauge}, nil } // LoggedCounter returns a Counter for tracking logged messages func (c *LoggerMetricsCollector) LoggedCounter(target string) (logr.Counter, error) { counter := prometheus.NewCounter(prometheus.CounterOpts{ Namespace: MetricsNamespace, Subsystem: "logging", Name: "logged_total", Help: "Total logged messages", ConstLabels: prometheus.Labels{"target": target}, }) _ = c.metrics.registry.Register(counter) return &simpleCounter{counter: counter}, nil } // ErrorCounter returns a Counter for tracking logging errors func (c *LoggerMetricsCollector) ErrorCounter(target string) (logr.Counter, error) { counter := prometheus.NewCounter(prometheus.CounterOpts{ Namespace: MetricsNamespace, Subsystem: "logging", Name: "errors_total", Help: "Total logging errors", ConstLabels: prometheus.Labels{"target": target}, }) _ = c.metrics.registry.Register(counter) return &simpleCounter{counter: counter}, nil } // DroppedCounter returns a Counter for tracking dropped messages func (c *LoggerMetricsCollector) DroppedCounter(target string) (logr.Counter, error) { counter := prometheus.NewCounter(prometheus.CounterOpts{ Namespace: MetricsNamespace, Subsystem: "logging", Name: "dropped_total", Help: "Total dropped messages", ConstLabels: prometheus.Labels{"target": target}, }) _ = c.metrics.registry.Register(counter) return &simpleCounter{counter: counter}, nil } // BlockedCounter returns a Counter for tracking blocked messages func (c *LoggerMetricsCollector) BlockedCounter(target string) (logr.Counter, error) { counter := prometheus.NewCounter(prometheus.CounterOpts{ Namespace: MetricsNamespace, Subsystem: "logging", Name: "blocked_total", Help: "Total blocked messages", ConstLabels: prometheus.Labels{"target": target}, }) _ = c.metrics.registry.Register(counter) return &simpleCounter{counter: counter}, nil }