// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. // See LICENSE.txt for license information. package platform import ( "crypto/ecdsa" "errors" "fmt" "hash/maphash" "net/http" "runtime" "strconv" "sync" "sync/atomic" "time" "github.com/mattermost/mattermost/server/public/model" "github.com/mattermost/mattermost/server/public/plugin" "github.com/mattermost/mattermost/server/public/shared/mlog" "github.com/mattermost/mattermost/server/v8/channels/app/featureflag" "github.com/mattermost/mattermost/server/v8/channels/jobs" "github.com/mattermost/mattermost/server/v8/channels/store" "github.com/mattermost/mattermost/server/v8/channels/store/localcachelayer" "github.com/mattermost/mattermost/server/v8/channels/store/retrylayer" "github.com/mattermost/mattermost/server/v8/channels/store/searchlayer" "github.com/mattermost/mattermost/server/v8/channels/store/sqlstore" "github.com/mattermost/mattermost/server/v8/channels/store/timerlayer" "github.com/mattermost/mattermost/server/v8/config" "github.com/mattermost/mattermost/server/v8/einterfaces" "github.com/mattermost/mattermost/server/v8/platform/services/cache" "github.com/mattermost/mattermost/server/v8/platform/services/searchengine" "github.com/mattermost/mattermost/server/v8/platform/shared/filestore" ) // PlatformService is the service for the platform related tasks. It is // responsible for non-entity related functionalities that are required // by a product such as database access, configuration access, licensing etc. type PlatformService struct { sqlStore *sqlstore.SqlStore Store store.Store newStore func() (store.Store, error) storeOptions []sqlstore.Option WebSocketRouter *WebSocketRouter configStore *config.Store filestore filestore.FileBackend exportFilestore filestore.FileBackend // Channel for batching status updates statusUpdateChan chan *model.Status statusUpdateExitSignal chan struct{} statusUpdateDoneSignal chan struct{} cacheProvider cache.Provider statusCache cache.Cache sessionCache cache.Cache asymmetricSigningKey atomic.Pointer[ecdsa.PrivateKey] clientConfig atomic.Value clientConfigHash atomic.Value limitedClientConfig atomic.Value isFirstUserAccountLock sync.Mutex isFirstUserAccount atomic.Bool logger *mlog.Logger startMetrics bool metrics *platformMetrics metricsIFace einterfaces.MetricsInterface featureFlagSynchronizerMutex sync.Mutex featureFlagSynchronizer *featureflag.Synchronizer featureFlagStop chan struct{} featureFlagStopped chan struct{} licenseValue atomic.Pointer[model.License] clientLicenseValue atomic.Value licenseListeners map[string]func(*model.License, *model.License) licenseManager einterfaces.LicenseInterface telemetryId string configListenerId string licenseListenerId string clusterLeaderListeners sync.Map clusterIFace einterfaces.ClusterInterface Busy *Busy SearchEngine *searchengine.Broker searchConfigListenerId string searchLicenseListenerId string ldapDiagnostic einterfaces.LdapDiagnosticInterface Jobs *jobs.JobServer hubs []*Hub hashSeed maphash.Seed goroutineCount int32 goroutineExitSignal chan struct{} goroutineBuffered chan struct{} additionalClusterHandlers map[model.ClusterEvent]einterfaces.ClusterMessageHandler shareChannelServiceMux sync.RWMutex sharedChannelService SharedChannelServiceIFace pluginEnv HookRunner // This is a test mode setting used to enable Redis // without a license. forceEnableRedis bool pdpService einterfaces.PolicyDecisionPointInterface } type HookRunner interface { RunMultiHook(hookRunnerFunc func(hooks plugin.Hooks, _ *model.Manifest) bool, hookId int) GetPluginsEnvironment() *plugin.Environment } // New creates a new PlatformService. func New(sc ServiceConfig, options ...Option) (*PlatformService, error) { // Step 0: Create the PlatformService. // ConfigStore is and should be handled on a upper level. ps := &PlatformService{ Store: sc.Store, clusterIFace: sc.Cluster, hashSeed: maphash.MakeSeed(), goroutineExitSignal: make(chan struct{}, 1), goroutineBuffered: make(chan struct{}, runtime.NumCPU()), WebSocketRouter: &WebSocketRouter{ handlers: make(map[string]webSocketHandler), }, licenseListeners: map[string]func(*model.License, *model.License){}, additionalClusterHandlers: map[model.ClusterEvent]einterfaces.ClusterMessageHandler{}, statusUpdateChan: make(chan *model.Status, statusUpdateBufferSize), statusUpdateExitSignal: make(chan struct{}), statusUpdateDoneSignal: make(chan struct{}), } // Assume the first user account has not been created yet. A call to the DB will later check if this is really the case. ps.isFirstUserAccount.Store(true) // Apply options, some of the options overrides the default config actually. for _, option := range options { if err2 := option(ps); err2 != nil { return nil, fmt.Errorf("failed to apply option: %w", err2) } } // the config store is not set, we need to create a new one if ps.configStore == nil { innerStore, err := config.NewFileStore("config.json", true) if err != nil { return nil, fmt.Errorf("failed to load config from file: %w", err) } configStore, err := config.NewStoreFromBacking(innerStore, nil, false) if err != nil { return nil, fmt.Errorf("failed to load config from file: %w", err) } ps.configStore = configStore } // Step 1: Start logging. err := ps.initLogging() if err != nil { return nil, fmt.Errorf("failed to initialize logging: %w", err) } ps.Log().Info("Server is initializing...", mlog.String("go_version", runtime.Version())) logCurrentVersion := fmt.Sprintf("Current version is %v (%v/%v/%v/%v)", model.CurrentVersion, model.BuildNumber, model.BuildDate, model.BuildHash, model.BuildHashEnterprise) ps.Log().Info( logCurrentVersion, mlog.String("current_version", model.CurrentVersion), mlog.String("build_number", model.BuildNumber), mlog.String("build_date", model.BuildDate), mlog.String("build_hash", model.BuildHash), mlog.String("build_hash_enterprise", model.BuildHashEnterprise), mlog.String("service_environment", model.GetServiceEnvironment()), ) if model.BuildEnterpriseReady == "true" { isTrial := false if licence := ps.License(); licence != nil { isTrial = licence.IsTrial } ps.Log().Info( "Enterprise Build", mlog.Bool("enterprise_build", true), mlog.Bool("is_trial", isTrial), ) } else { ps.Log().Info("Team Edition Build", mlog.Bool("enterprise_build", false)) } // Step 2: Cache provider. cacheConfig := ps.configStore.Get().CacheSettings if *cacheConfig.CacheType == model.CacheTypeLRU { ps.cacheProvider = cache.NewProvider() } else if *cacheConfig.CacheType == model.CacheTypeRedis { ps.cacheProvider, err = cache.NewRedisProvider( &cache.RedisOptions{ RedisAddr: *cacheConfig.RedisAddress, RedisPassword: *cacheConfig.RedisPassword, RedisDB: *cacheConfig.RedisDB, RedisCachePrefix: *cacheConfig.RedisCachePrefix, DisableCache: *cacheConfig.DisableClientCache, }, ) } if err != nil { return nil, fmt.Errorf("unable to create cache provider: %w", err) } res, err := ps.cacheProvider.Connect() if err != nil { return nil, fmt.Errorf("unable to connect to cache provider: %w", err) } ps.Log().Info("Successfully connected to cache backend", mlog.String("backend", *cacheConfig.CacheType), mlog.String("result", res)) // Step 3: Search Engine searchEngine := searchengine.NewBroker(ps.Config()) ps.SearchEngine = searchEngine // Step 4: Init Enterprise // Depends on step 3 (s.SearchEngine must be non-nil) ps.initEnterprise() // Step 5: Init Metrics if metricsInterfaceFn != nil && ps.metricsIFace == nil { // if the metrics interface is set by options, do not override it ps.metricsIFace = metricsInterfaceFn(ps, *ps.configStore.Get().SqlSettings.DriverName, *ps.configStore.Get().SqlSettings.DataSource) } ps.cacheProvider.SetMetrics(ps.metricsIFace) // Step 6: Store. // Depends on Step 0 (config), 1 (cacheProvider), 3 (search engine), 5 (metrics) and cluster. if ps.newStore == nil { ps.newStore = func() (store.Store, error) { // The layer cake is as follows: (From bottom to top) // SQL layer // | // Retry layer // | // Search layer // | // Timer layer // | // Cache layer ps.sqlStore, err = sqlstore.New(ps.Config().SqlSettings, ps.Log(), ps.metricsIFace, ps.storeOptions...) if err != nil { return nil, err } searchStore := searchlayer.NewSearchLayer( retrylayer.New(ps.sqlStore), ps.SearchEngine, ps.Config(), ) ps.AddConfigListener(func(prevCfg, cfg *model.Config) { searchStore.UpdateConfig(cfg) }) lcl, err2 := localcachelayer.NewLocalCacheLayer( timerlayer.New(searchStore, ps.metricsIFace), ps.metricsIFace, ps.clusterIFace, ps.cacheProvider, ps.Log(), ) if err2 != nil { return nil, fmt.Errorf("cannot create local cache layer: %w", err2) } license := ps.License() ps.sqlStore.UpdateLicense(license) ps.AddLicenseListener(func(oldLicense, newLicense *model.License) { ps.sqlStore.UpdateLicense(newLicense) }) return lcl, nil } } ps.Store, err = ps.newStore() if err != nil { return nil, fmt.Errorf("cannot create store: %w", err) } // Step 7: initialize status and session cache. // We need to do this because ps.LoadLicense() called in step 8, could // end up calling InvalidateAllCaches, so the status and session caches // need to be initialized before that. // Note: we hardcode the session and status cache to LRU because they lead // to a lot of SCAN calls in case of Redis. We could potentially have a // reverse mapping to avoid the scan, but this needs more complicated code. // Leaving this for now. ps.statusCache, err = cache.NewProvider().NewCache(&cache.CacheOptions{ Name: "Status", Size: model.StatusCacheSize, Striped: true, StripedBuckets: max(runtime.NumCPU()-1, 1), DefaultExpiry: 30 * time.Minute, }) if err != nil { return nil, fmt.Errorf("unable to create status cache: %w", err) } ps.sessionCache, err = cache.NewProvider().NewCache(&cache.CacheOptions{ Name: "Session", Size: model.SessionCacheSize, Striped: true, StripedBuckets: max(runtime.NumCPU()-1, 1), }) if err != nil { return nil, fmt.Errorf("could not create session cache: %w", err) } // Step 8: Init License if model.BuildEnterpriseReady == "true" { ps.LoadLicense() } license := ps.License() // This is a hack because ideally we wouldn't even have started the Redis client // if the license didn't have clustering. But there's an intricate deadlock // where license cannot be loaded before store, and store cannot be loaded before // cache. So loading license before loading cache is an uphill battle. if (license == nil || !*license.Features.Cluster) && *cacheConfig.CacheType == model.CacheTypeRedis && !ps.forceEnableRedis { return nil, fmt.Errorf("Redis cannot be used in an instance without a license or a license without clustering") } // Step 9: Initialize filestore if ps.filestore == nil { insecure := ps.Config().ServiceSettings.EnableInsecureOutgoingConnections backend, err2 := filestore.NewFileBackend(filestore.NewFileBackendSettingsFromConfig(&ps.Config().FileSettings, license != nil && *license.Features.Compliance, insecure != nil && *insecure)) if err2 != nil { return nil, fmt.Errorf("failed to initialize filebackend: %w", err2) } ps.filestore = backend } if ps.exportFilestore == nil { ps.exportFilestore = ps.filestore if *ps.Config().FileSettings.DedicatedExportStore { mlog.Info("Setting up dedicated export filestore", mlog.String("driver_name", *ps.Config().FileSettings.ExportDriverName)) backend, errFileBack := filestore.NewExportFileBackend(filestore.NewExportFileBackendSettingsFromConfig(&ps.Config().FileSettings, license != nil && *license.Features.Compliance, false)) if errFileBack != nil { return nil, fmt.Errorf("failed to initialize export filebackend: %w", errFileBack) } ps.exportFilestore = backend } } // Step 10: Init Metrics Server depends on step 6 (store) and 8 (license) if ps.startMetrics { if mErr := ps.resetMetrics(); mErr != nil { return nil, mErr } ps.configStore.AddListener(func(oldCfg, newCfg *model.Config) { if *oldCfg.MetricsSettings.Enable != *newCfg.MetricsSettings.Enable || *oldCfg.MetricsSettings.ListenAddress != *newCfg.MetricsSettings.ListenAddress { if mErr := ps.resetMetrics(); mErr != nil { mlog.Warn("Failed to reset metrics", mlog.Err(mErr)) } } }) } // Step 11: Init AsymmetricSigningKey depends on step 6 (store) if err = ps.EnsureAsymmetricSigningKey(); err != nil { return nil, fmt.Errorf("unable to ensure asymmetric signing key: %w", err) } ps.Busy = NewBusy(ps.clusterIFace) // Enable developer settings and mmctl local mode if this is a "dev" build if model.BuildNumber == "dev" { ps.UpdateConfig(func(cfg *model.Config) { *cfg.ServiceSettings.EnableDeveloper = true *cfg.ServiceSettings.EnableLocalMode = true }) } ps.AddLicenseListener(func(oldLicense, newLicense *model.License) { wasLicensed := (oldLicense != nil && *oldLicense.Features.Metrics) || (model.BuildNumber == "dev") isLicensed := (newLicense != nil && *newLicense.Features.Metrics) || (model.BuildNumber == "dev") if wasLicensed == isLicensed || !ps.startMetrics { return } if err := ps.RestartMetrics(); err != nil { ps.logger.Error("Failed to reset metrics server", mlog.Err(err)) } }) if err := ps.SearchEngine.UpdateConfig(ps.Config()); err != nil { ps.logger.Error("Failed to update search engine config", mlog.Err(err)) } searchConfigListenerId, searchLicenseListenerId := ps.StartSearchEngine() ps.searchConfigListenerId = searchConfigListenerId ps.searchLicenseListenerId = searchLicenseListenerId return ps, nil } func (ps *PlatformService) Start(broadcastHooks map[string]BroadcastHook) error { // Start the status update processor. // Must be done before hub start. go ps.processStatusUpdates() ps.hubStart(broadcastHooks) ps.configListenerId = ps.AddConfigListener(func(_, _ *model.Config) { ps.regenerateClientConfig() message := model.NewWebSocketEvent(model.WebsocketEventConfigChanged, "", "", "", nil, "") message.Add("config", ps.ClientConfigWithComputed()) ps.Go(func() { ps.Publish(message) }) if err := ps.ReconfigureLogger(); err != nil { mlog.Error("Error re-configuring logging after config change", mlog.Err(err)) return } }) ps.licenseListenerId = ps.AddLicenseListener(func(oldLicense, newLicense *model.License) { ps.regenerateClientConfig() message := model.NewWebSocketEvent(model.WebsocketEventLicenseChanged, "", "", "", nil, "") message.Add("license", ps.GetSanitizedClientLicense()) ps.Publish(message) }) return nil } func (ps *PlatformService) ShutdownMetrics() error { if ps.metrics != nil { return ps.metrics.stopMetricsServer() } return nil } func (ps *PlatformService) ShutdownConfig() error { ps.RemoveConfigListener(ps.configListenerId) if ps.configStore != nil { err := ps.configStore.Close() if err != nil { return fmt.Errorf("failed to close config store: %w", err) } } return nil } func (ps *PlatformService) SetTelemetryId(id string) { ps.telemetryId = id ps.PostTelemetryIdHook() } // PostTelemetryIdHook triggers necessary events to propagate telemtery ID func (ps *PlatformService) PostTelemetryIdHook() { ps.regenerateClientConfig() } func (ps *PlatformService) SetLogger(logger *mlog.Logger) { ps.logger = logger } func (ps *PlatformService) initEnterprise() { if clusterInterface != nil && ps.clusterIFace == nil { ps.clusterIFace = clusterInterface(ps) } if elasticsearchInterface != nil { ps.SearchEngine.RegisterElasticsearchEngine(elasticsearchInterface(ps)) } if ldapDiagnosticInterface != nil { ps.ldapDiagnostic = ldapDiagnosticInterface(ps) } if licenseInterface != nil { ps.licenseManager = licenseInterface(ps) } if accessControlServiceInterface != nil { ps.pdpService = accessControlServiceInterface(ps) } } func (ps *PlatformService) TotalWebsocketConnections() int { // This method is only called after the hub is initialized. // Therefore, no mutex is needed to protect s.hubs. count := int64(0) for _, hub := range ps.hubs { count = count + atomic.LoadInt64(&hub.connectionCount) } return int(count) } func (ps *PlatformService) Shutdown() error { ps.HubStop() // Shutdown status processor. // Must be done after hub shutdown. close(ps.statusUpdateExitSignal) // wait for it to be stopped. <-ps.statusUpdateDoneSignal ps.RemoveLicenseListener(ps.licenseListenerId) // we need to wait the goroutines to finish before closing the store // and this needs to be called after hub stop because hub generates goroutines // when it is active. If we wait first we have no mechanism to prevent adding // more go routines hence they still going to be invoked. ps.waitForGoroutines() if ps.Store != nil { ps.Store.Close() } if ps.cacheProvider != nil { if err := ps.cacheProvider.Close(); err != nil { return fmt.Errorf("unable to cleanly shutdown cache: %w", err) } } return nil } func (ps *PlatformService) CacheProvider() cache.Provider { return ps.cacheProvider } // SetSqlStore is used for plugin testing func (ps *PlatformService) SetSqlStore(s *sqlstore.SqlStore) { ps.sqlStore = s } func (ps *PlatformService) SetSharedChannelService(s SharedChannelServiceIFace) { ps.shareChannelServiceMux.Lock() defer ps.shareChannelServiceMux.Unlock() ps.sharedChannelService = s } func (ps *PlatformService) GetSharedChannelService() SharedChannelServiceIFace { ps.shareChannelServiceMux.RLock() defer ps.shareChannelServiceMux.RUnlock() return ps.sharedChannelService } func (ps *PlatformService) SetPluginsEnvironment(runner HookRunner) { ps.pluginEnv = runner } // GetPluginStatuses meant to be used by cluster implementation func (ps *PlatformService) GetPluginStatuses() (model.PluginStatuses, *model.AppError) { if ps.pluginEnv == nil || ps.pluginEnv.GetPluginsEnvironment() == nil { return nil, model.NewAppError("GetPluginStatuses", "app.plugin.disabled.app_error", nil, "", http.StatusNotImplemented) } pluginStatuses, err := ps.pluginEnv.GetPluginsEnvironment().Statuses() if err != nil { return nil, model.NewAppError("GetPluginStatuses", "app.plugin.get_statuses.app_error", nil, "", http.StatusInternalServerError).Wrap(err) } // Add our cluster ID for _, status := range pluginStatuses { if ps.Cluster() != nil { status.ClusterId = ps.Cluster().GetClusterId() } else { status.ClusterId = "" } } return pluginStatuses, nil } func (ps *PlatformService) getPluginManifests() ([]*model.Manifest, error) { if ps.pluginEnv == nil { return nil, errors.New("plugin environment not initialized") } pluginsEnvironment := ps.pluginEnv.GetPluginsEnvironment() if pluginsEnvironment == nil { return nil, model.NewAppError("getPluginManifests", "app.plugin.disabled.app_error", nil, "", http.StatusNotImplemented) } plugins, err := pluginsEnvironment.Available() if err != nil { return nil, fmt.Errorf("failed to get list of available plugins: %w", err) } manifests := make([]*model.Manifest, len(plugins)) for i := range plugins { manifests[i] = plugins[i].Manifest } return manifests, nil } func (ps *PlatformService) FileBackend() filestore.FileBackend { return ps.filestore } func (ps *PlatformService) ExportFileBackend() filestore.FileBackend { return ps.exportFilestore } func (ps *PlatformService) LdapDiagnostic() einterfaces.LdapDiagnosticInterface { return ps.ldapDiagnostic } // DatabaseTypeAndSchemaVersion returns the Database type (postgres or mysql) and current version of the schema func (ps *PlatformService) DatabaseTypeAndSchemaVersion() (string, string, error) { schemaVersion, err := ps.Store.GetDBSchemaVersion() if err != nil { return "", "", err } return model.SafeDereference(ps.Config().SqlSettings.DriverName), strconv.Itoa(schemaVersion), nil }