ec1f89217a
Full Mattermost server source with integrated Community Enterprise features. Includes vendor directory for offline/air-gapped builds. Structure: - enterprise-impl/: Enterprise feature implementations - enterprise-community/: Init files that register implementations - enterprise/: Bridge imports (community_imports.go) - vendor/: All dependencies for offline builds Build (online): go build ./cmd/mattermost Build (offline/air-gapped): go build -mod=vendor ./cmd/mattermost 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
1318 lines
42 KiB
Go
1318 lines
42 KiB
Go
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
|
|
// See LICENSE.txt for license information.
|
|
|
|
package app
|
|
|
|
import (
|
|
"archive/zip"
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"os"
|
|
"path"
|
|
"path/filepath"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/pkg/errors"
|
|
|
|
"github.com/mattermost/mattermost/server/public/model"
|
|
"github.com/mattermost/mattermost/server/public/shared/mlog"
|
|
"github.com/mattermost/mattermost/server/public/shared/request"
|
|
"github.com/mattermost/mattermost/server/v8/channels/app/imports"
|
|
"github.com/mattermost/mattermost/server/v8/channels/store"
|
|
"github.com/mattermost/mattermost/server/v8/platform/shared/filestore"
|
|
)
|
|
|
|
const warningsFilename = "warnings.txt"
|
|
|
|
// We use this map to identify the exportable preferences.
|
|
// Here we link the preference category and name, to the name of the relevant field in the import struct.
|
|
var exportablePreferences = map[imports.ComparablePreference]string{
|
|
{
|
|
Category: model.PreferenceCategoryTheme,
|
|
Name: "",
|
|
}: "Theme",
|
|
{
|
|
Category: model.PreferenceCategoryAdvancedSettings,
|
|
Name: "feature_enabled_markdown_preview",
|
|
}: "UseMarkdownPreview",
|
|
{
|
|
Category: model.PreferenceCategoryAdvancedSettings,
|
|
Name: "formatting",
|
|
}: "UseFormatting",
|
|
{
|
|
Category: model.PreferenceCategoryAdvancedSettings,
|
|
Name: "send_on_ctrl_enter",
|
|
}: "SendOnCtrlEnter",
|
|
{
|
|
Category: model.PreferenceCategoryAdvancedSettings,
|
|
Name: "code_block_ctrl_enter",
|
|
}: "CodeBlockCtrlEnter",
|
|
{
|
|
Category: model.PreferenceCategoryAdvancedSettings,
|
|
Name: "join_leave",
|
|
}: "ShowJoinLeave",
|
|
{
|
|
Category: model.PreferenceCategoryAdvancedSettings,
|
|
Name: "unread_scroll_position",
|
|
}: "ShowUnreadScrollPosition",
|
|
{
|
|
Category: model.PreferenceCategoryAdvancedSettings,
|
|
Name: "sync_drafts",
|
|
}: "SyncDrafts",
|
|
{
|
|
Category: model.PreferenceCategorySidebarSettings,
|
|
Name: model.PreferenceNameShowUnreadSection,
|
|
}: "ShowUnreadSection",
|
|
{
|
|
Category: model.PreferenceCategorySidebarSettings,
|
|
Name: model.PreferenceLimitVisibleDmsGms,
|
|
}: "LimitVisibleDmsGms",
|
|
{
|
|
Category: model.PreferenceCategoryDisplaySettings,
|
|
Name: model.PreferenceNameUseMilitaryTime,
|
|
}: "UseMilitaryTime",
|
|
{
|
|
Category: model.PreferenceCategoryDisplaySettings,
|
|
Name: model.PreferenceNameCollapseSetting,
|
|
}: "CollapsePreviews",
|
|
{
|
|
Category: model.PreferenceCategoryDisplaySettings,
|
|
Name: model.PreferenceNameMessageDisplay,
|
|
}: "MessageDisplay",
|
|
{
|
|
Category: model.PreferenceCategoryDisplaySettings,
|
|
Name: model.PreferenceNameChannelDisplayMode,
|
|
}: "ChannelDisplayMode",
|
|
{
|
|
Category: model.PreferenceCategoryDisplaySettings,
|
|
Name: model.PreferenceNameCollapseConsecutive,
|
|
}: "CollapseConsecutive",
|
|
{
|
|
Category: model.PreferenceCategoryDisplaySettings,
|
|
Name: model.PreferenceNameColorizeUsernames,
|
|
}: "ColorizeUsernames",
|
|
{
|
|
Category: model.PreferenceCategoryDisplaySettings,
|
|
Name: model.PreferenceNameNameFormat,
|
|
}: "NameFormat",
|
|
{
|
|
Category: model.PreferenceCategoryTutorialSteps,
|
|
Name: "",
|
|
}: "TutorialStep",
|
|
{
|
|
Category: model.PreferenceCategoryNotifications,
|
|
Name: model.PreferenceNameEmailInterval,
|
|
}: "EmailInterval",
|
|
}
|
|
|
|
func (a *App) BulkExport(rctx request.CTX, writer io.Writer, outPath string, job *model.Job, opts model.BulkExportOpts) *model.AppError {
|
|
var zipWr *zip.Writer
|
|
if opts.CreateArchive {
|
|
var err error
|
|
zipWr = zip.NewWriter(writer)
|
|
defer func() {
|
|
if err = zipWr.Close(); err != nil {
|
|
rctx.Logger().Error("Error closing zip writer", mlog.Err(err))
|
|
}
|
|
}()
|
|
writer, err = zipWr.Create("import.jsonl")
|
|
if err != nil {
|
|
return model.NewAppError("BulkExport", "app.export.zip_create.error",
|
|
nil, "", http.StatusInternalServerError).Wrap(err)
|
|
}
|
|
}
|
|
|
|
if job == nil {
|
|
job = &model.Job{
|
|
Data: make(model.StringMap),
|
|
}
|
|
} else if job.Data == nil {
|
|
job.Data = make(model.StringMap)
|
|
}
|
|
|
|
rctx.Logger().Info("Bulk export: exporting version")
|
|
if err := a.exportVersion(writer); err != nil {
|
|
return err
|
|
}
|
|
|
|
if opts.IncludeRolesAndSchemes {
|
|
if err := a.exportRolesAndSchemes(rctx, job, writer); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
|
|
rctx.Logger().Info("Bulk export: exporting teams")
|
|
teamNames, appErr := a.exportAllTeams(rctx, job, writer)
|
|
if appErr != nil {
|
|
return appErr
|
|
}
|
|
|
|
rctx.Logger().Info("Bulk export: exporting channels")
|
|
if appErr = a.exportAllChannels(rctx, job, writer, teamNames, opts.IncludeArchivedChannels); appErr != nil {
|
|
return appErr
|
|
}
|
|
|
|
rctx.Logger().Info("Bulk export: exporting users")
|
|
profilePictures, appErr := a.exportAllUsers(rctx, job, writer, opts.IncludeArchivedChannels, opts.IncludeProfilePictures)
|
|
if appErr != nil {
|
|
return appErr
|
|
}
|
|
|
|
rctx.Logger().Info("Bulk export: exporting bots")
|
|
botPPs, appErr := a.exportAllBots(rctx, job, writer, opts.IncludeProfilePictures)
|
|
if appErr != nil {
|
|
return appErr
|
|
}
|
|
profilePictures = append(profilePictures, botPPs...)
|
|
|
|
rctx.Logger().Info("Bulk export: exporting posts")
|
|
attachments, appErr := a.exportAllPosts(rctx, job, writer, opts.IncludeAttachments, opts.IncludeArchivedChannels)
|
|
if appErr != nil {
|
|
return appErr
|
|
}
|
|
|
|
rctx.Logger().Info("Bulk export: exporting emoji")
|
|
emojiPaths, appErr := a.exportCustomEmoji(rctx, job, writer, outPath, "exported_emoji", !opts.CreateArchive)
|
|
if appErr != nil {
|
|
return appErr
|
|
}
|
|
|
|
rctx.Logger().Info("Bulk export: exporting direct channels")
|
|
if appErr = a.exportAllDirectChannels(rctx, job, writer, opts.IncludeArchivedChannels); appErr != nil {
|
|
return appErr
|
|
}
|
|
|
|
rctx.Logger().Info("Bulk export: exporting direct posts")
|
|
directAttachments, appErr := a.exportAllDirectPosts(rctx, job, writer, opts.IncludeAttachments, opts.IncludeArchivedChannels)
|
|
if appErr != nil {
|
|
return appErr
|
|
}
|
|
|
|
if opts.IncludeAttachments {
|
|
rctx.Logger().Info("Bulk export: exporting file attachments")
|
|
warnings, appErr := a.exportAttachments(rctx, attachments, outPath, zipWr)
|
|
if appErr != nil {
|
|
return appErr
|
|
}
|
|
|
|
rctx.Logger().Info("Bulk export: exporting direct file attachments")
|
|
newWarnings, appErr := a.exportAttachments(rctx, directAttachments, outPath, zipWr)
|
|
if appErr != nil {
|
|
return appErr
|
|
}
|
|
warnings = append(warnings, newWarnings...)
|
|
|
|
totalExportedEmojis := 0
|
|
emojisLen := len(emojiPaths)
|
|
rctx.Logger().Info("Bulk export: exporting custom emojis")
|
|
for _, emojiPath := range emojiPaths {
|
|
if err := a.exportFile(rctx, outPath, emojiPath, zipWr); err != nil {
|
|
rctx.Logger().Warn("Unable to export emoji", mlog.String("emoji_path", emojiPath), mlog.Err(err))
|
|
} else {
|
|
totalExportedEmojis++
|
|
}
|
|
if totalExportedEmojis%10 == 0 {
|
|
rctx.Logger().Info("Bulk export: exporting emojis progress", mlog.Int("total_successfully_exported_emojis", totalExportedEmojis), mlog.Int("total_emojis_to_export", emojisLen))
|
|
}
|
|
}
|
|
|
|
if len(warnings) > 0 {
|
|
warningsFile, _ := zipWr.Create(warningsFilename)
|
|
for _, warning := range warnings {
|
|
_, err := warningsFile.Write([]byte(warning + "\n"))
|
|
if err != nil {
|
|
return model.NewAppError("BulkExport", "app.export.zip_create.error",
|
|
nil, "", http.StatusInternalServerError).Wrap(err)
|
|
}
|
|
}
|
|
updateJobProgress(rctx.Logger(), a.Srv().Store(), job, "num_warnings", len(warnings))
|
|
}
|
|
|
|
updateJobProgress(rctx.Logger(), a.Srv().Store(), job, "attachments_exported", len(attachments)+len(directAttachments)+len(emojiPaths))
|
|
}
|
|
|
|
if opts.IncludeProfilePictures {
|
|
rctx.Logger().Info("Bulk export: exporting profile pictures")
|
|
for _, profilePicture := range profilePictures {
|
|
if err := a.exportFile(rctx, outPath, profilePicture, zipWr); err != nil {
|
|
rctx.Logger().Warn("Unable to export profile picture", mlog.String("profile_picture", profilePicture), mlog.Err(err))
|
|
}
|
|
}
|
|
updateJobProgress(rctx.Logger(), a.Srv().Store(), job, "profile_pictures_exported", len(profilePictures))
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func (a *App) exportAttachments(rctx request.CTX, attachments []imports.AttachmentImportData, outPath string,
|
|
zipWr *zip.Writer) ([]string, *model.AppError) {
|
|
totalExportedFiles := 0
|
|
attachmentsLen := len(attachments)
|
|
var warnings []string
|
|
for _, attachment := range attachments {
|
|
if err := a.exportFile(rctx, outPath, *attachment.Path, zipWr); err != nil {
|
|
rctx.Logger().Warn("Unable to export file attachment", mlog.String("attachment_path", *attachment.Path), mlog.Err(err))
|
|
warnings = append(warnings, fmt.Sprintf("Unable to export file attachment, attachment path: %s , error: %s", *attachment.Path, err.Error()))
|
|
} else {
|
|
totalExportedFiles++
|
|
}
|
|
if totalExportedFiles%10 == 0 {
|
|
rctx.Logger().Info("Bulk export: exporting file attachments progress", mlog.Int("total_successfully_exported_files", totalExportedFiles), mlog.Int("total_files_to_export", attachmentsLen))
|
|
}
|
|
}
|
|
return warnings, nil
|
|
}
|
|
|
|
func (a *App) exportWriteLine(w io.Writer, line *imports.LineImportData) *model.AppError {
|
|
b, err := json.Marshal(line)
|
|
if err != nil {
|
|
return model.NewAppError("BulkExport", "app.export.export_write_line.json_marshall.error", nil, "", http.StatusBadRequest).Wrap(err)
|
|
}
|
|
|
|
if _, err := w.Write(append(b, '\n')); err != nil {
|
|
return model.NewAppError("BulkExport", "app.export.export_write_line.io_writer.error", nil, "", http.StatusBadRequest).Wrap(err)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func (a *App) exportVersion(writer io.Writer) *model.AppError {
|
|
version := 1
|
|
|
|
info := &imports.VersionInfoImportData{
|
|
Generator: "mattermost-server",
|
|
Version: fmt.Sprintf("%s (%s, enterprise: %s)", model.CurrentVersion, model.BuildHash, model.BuildEnterpriseReady),
|
|
Created: time.Now().Format(time.RFC3339Nano),
|
|
}
|
|
|
|
versionLine := &imports.LineImportData{
|
|
Type: "version",
|
|
Version: &version,
|
|
Info: info,
|
|
}
|
|
|
|
return a.exportWriteLine(writer, versionLine)
|
|
}
|
|
|
|
func (a *App) exportRolesAndSchemes(rctx request.CTX, job *model.Job, writer io.Writer) *model.AppError {
|
|
// We export schemes first since they'll already include their attached roles
|
|
// which we map to avoid exporting them twice later in exportRoles.
|
|
schemeRolesMap := make(map[string]bool)
|
|
|
|
roles, appErr := a.Srv().Store().Role().GetAll()
|
|
if appErr != nil {
|
|
return model.NewAppError("exportRolesAndSchemes", "app.role.get_all.app_error", nil, "", http.StatusInternalServerError).Wrap(appErr)
|
|
}
|
|
|
|
rctx.Logger().Info("Bulk export: exporting team schemes")
|
|
if err := a.exportSchemes(rctx, job, writer, model.SchemeScopeTeam, schemeRolesMap, roles); err != nil {
|
|
return err
|
|
}
|
|
|
|
rctx.Logger().Info("Bulk export: exporting channel schemes")
|
|
if err := a.exportSchemes(rctx, job, writer, model.SchemeScopeChannel, schemeRolesMap, roles); err != nil {
|
|
return err
|
|
}
|
|
|
|
rctx.Logger().Info("Bulk export: exporting roles")
|
|
if err := a.exportRoles(rctx, job, writer, schemeRolesMap, roles); err != nil {
|
|
return err
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func (a *App) exportRoles(rctx request.CTX, job *model.Job, writer io.Writer, schemeRoles map[string]bool, allRoles []*model.Role) *model.AppError {
|
|
var cnt int
|
|
for _, role := range allRoles {
|
|
// We skip any roles that will be included as part of custom schemes.
|
|
if !schemeRoles[role.Name] {
|
|
if err := a.exportWriteLine(writer, importLineFromRole(role)); err != nil {
|
|
return err
|
|
}
|
|
cnt++
|
|
}
|
|
}
|
|
|
|
updateJobProgress(rctx.Logger(), a.Srv().Store(), job, "roles_exported", cnt)
|
|
|
|
return nil
|
|
}
|
|
|
|
func (a *App) exportSchemes(rctx request.CTX, job *model.Job, writer io.Writer, scope string, schemeRolesMap map[string]bool, allRoles []*model.Role) *model.AppError {
|
|
rolesMap := make(map[string]*model.Role, len(allRoles))
|
|
for _, role := range allRoles {
|
|
rolesMap[role.Name] = role
|
|
}
|
|
|
|
var cnt int
|
|
pageSize := 100
|
|
|
|
for {
|
|
schemes, err := a.Srv().Store().Scheme().GetAllPage(scope, cnt, pageSize)
|
|
if err != nil {
|
|
return model.NewAppError("exportSchemes", "app.scheme.get_all_page.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
|
|
}
|
|
|
|
for _, scheme := range schemes {
|
|
if ok := scheme.IsValid(); !ok {
|
|
return model.NewAppError("exportSchemes", "model.scheme.is_valid.app_error", nil, "", http.StatusInternalServerError)
|
|
}
|
|
|
|
if scheme.Scope == model.SchemeScopeTeam {
|
|
schemeRolesMap[scheme.DefaultTeamAdminRole] = true
|
|
schemeRolesMap[scheme.DefaultTeamUserRole] = true
|
|
schemeRolesMap[scheme.DefaultTeamGuestRole] = true
|
|
|
|
// Playbooks
|
|
// At the moment this is only needed to avoid exporting and
|
|
// importing spurious roles.
|
|
schemeRolesMap[scheme.DefaultPlaybookAdminRole] = true
|
|
schemeRolesMap[scheme.DefaultPlaybookMemberRole] = true
|
|
schemeRolesMap[scheme.DefaultRunAdminRole] = true
|
|
schemeRolesMap[scheme.DefaultRunMemberRole] = true
|
|
}
|
|
|
|
if scheme.Scope == model.SchemeScopeTeam || scheme.Scope == model.SchemeScopeChannel {
|
|
schemeRolesMap[scheme.DefaultChannelAdminRole] = true
|
|
schemeRolesMap[scheme.DefaultChannelUserRole] = true
|
|
schemeRolesMap[scheme.DefaultChannelGuestRole] = true
|
|
}
|
|
|
|
if err := a.exportWriteLine(writer, importLineFromScheme(scheme, rolesMap)); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
|
|
cnt += len(schemes)
|
|
|
|
updateJobProgress(rctx.Logger(), a.Srv().Store(), job, fmt.Sprintf("%s_schemes_exported", scope), cnt)
|
|
|
|
if len(schemes) < pageSize {
|
|
return nil
|
|
}
|
|
}
|
|
}
|
|
|
|
func (a *App) exportAllTeams(rctx request.CTX, job *model.Job, writer io.Writer) (map[string]bool, *model.AppError) {
|
|
afterId := strings.Repeat("0", 26)
|
|
teamNames := make(map[string]bool)
|
|
cnt := 0
|
|
for {
|
|
teams, err := a.Srv().Store().Team().GetAllForExportAfter(1000, afterId)
|
|
if err != nil {
|
|
return nil, model.NewAppError("exportAllTeams", "app.team.get_all.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
|
|
}
|
|
|
|
if len(teams) == 0 {
|
|
break
|
|
}
|
|
cnt += len(teams)
|
|
updateJobProgress(rctx.Logger(), a.Srv().Store(), job, "teams_exported", cnt)
|
|
|
|
for _, team := range teams {
|
|
afterId = team.Id
|
|
|
|
// Skip deleted.
|
|
if team.DeleteAt != 0 {
|
|
continue
|
|
}
|
|
teamNames[team.Name] = true
|
|
|
|
teamLine := importLineFromTeam(team)
|
|
if err := a.exportWriteLine(writer, teamLine); err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
}
|
|
|
|
return teamNames, nil
|
|
}
|
|
|
|
func (a *App) exportAllChannels(rctx request.CTX, job *model.Job, writer io.Writer, teamNames map[string]bool, withArchived bool) *model.AppError {
|
|
afterId := strings.Repeat("0", 26)
|
|
cnt := 0
|
|
for {
|
|
channels, err := a.Srv().Store().Channel().GetAllChannelsForExportAfter(1000, afterId)
|
|
if err != nil {
|
|
return model.NewAppError("exportAllChannels", "app.channel.get_all.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
|
|
}
|
|
|
|
if len(channels) == 0 {
|
|
break
|
|
}
|
|
cnt += len(channels)
|
|
updateJobProgress(rctx.Logger(), a.Srv().Store(), job, "channels_exported", cnt)
|
|
|
|
for _, channel := range channels {
|
|
afterId = channel.Id
|
|
|
|
// Skip deleted.
|
|
if channel.DeleteAt != 0 && !withArchived {
|
|
continue
|
|
}
|
|
// Skip channels on deleted teams.
|
|
if ok := teamNames[channel.TeamName]; !ok {
|
|
continue
|
|
}
|
|
|
|
channelLine := importLineFromChannel(channel)
|
|
if err := a.exportWriteLine(writer, channelLine); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func (a *App) exportAllUsers(rctx request.CTX, job *model.Job, writer io.Writer, includeArchivedChannels, includeProfilePictures bool) ([]string, *model.AppError) {
|
|
afterId := strings.Repeat("0", 26)
|
|
cnt := 0
|
|
profilePictures := []string{}
|
|
for {
|
|
users, err := a.Srv().Store().User().GetAllAfter(1000, afterId)
|
|
if err != nil {
|
|
return profilePictures, model.NewAppError("exportAllUsers", "app.user.get.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
|
|
}
|
|
|
|
if len(users) == 0 {
|
|
break
|
|
}
|
|
cnt += len(users)
|
|
updateJobProgress(rctx.Logger(), a.Srv().Store(), job, "users_exported", cnt)
|
|
|
|
for _, user := range users {
|
|
afterId = user.Id
|
|
|
|
// Skip bots as they are exported separately.
|
|
if user.IsBot {
|
|
continue
|
|
}
|
|
|
|
// Gathering here the exportable preferences to pass them on to importLineFromUser
|
|
exportedPrefs := make(map[string]*string)
|
|
allPrefs, err := a.GetPreferencesForUser(rctx, user.Id)
|
|
if err != nil {
|
|
return profilePictures, err
|
|
}
|
|
for _, pref := range allPrefs {
|
|
// We need to manage the special cases
|
|
// Here we manage Tutorial steps
|
|
if pref.Category == model.PreferenceCategoryTutorialSteps {
|
|
pref.Name = ""
|
|
// Then the email interval
|
|
} else if pref.Category == model.PreferenceCategoryNotifications && pref.Name == model.PreferenceNameEmailInterval {
|
|
switch pref.Value {
|
|
case model.PreferenceEmailIntervalNoBatchingSeconds:
|
|
pref.Value = model.PreferenceEmailIntervalImmediately
|
|
case model.PreferenceEmailIntervalFifteenAsSeconds:
|
|
pref.Value = model.PreferenceEmailIntervalFifteen
|
|
case model.PreferenceEmailIntervalHourAsSeconds:
|
|
pref.Value = model.PreferenceEmailIntervalHour
|
|
case "0":
|
|
pref.Value = ""
|
|
}
|
|
}
|
|
id, ok := exportablePreferences[imports.ComparablePreference{
|
|
Category: pref.Category,
|
|
Name: pref.Name,
|
|
}]
|
|
if ok {
|
|
prefPtr := pref.Value
|
|
if prefPtr != "" {
|
|
exportedPrefs[id] = &prefPtr
|
|
} else {
|
|
exportedPrefs[id] = nil
|
|
}
|
|
}
|
|
}
|
|
|
|
userLine := importLineFromUser(user, exportedPrefs)
|
|
|
|
if includeProfilePictures {
|
|
var pp string
|
|
pp, err = a.GetProfileImagePath(user)
|
|
if err != nil {
|
|
return profilePictures, err
|
|
}
|
|
if pp != "" {
|
|
userLine.User.ProfileImage = &pp
|
|
profilePictures = append(profilePictures, pp)
|
|
}
|
|
}
|
|
|
|
userLine.User.NotifyProps = a.buildUserNotifyProps(user.NotifyProps)
|
|
|
|
// Adding custom status
|
|
if cs := user.GetCustomStatus(); cs != nil {
|
|
userLine.User.CustomStatus = cs
|
|
}
|
|
|
|
// Do the Team Memberships.
|
|
members, err := a.buildUserTeamAndChannelMemberships(rctx, user.Id, includeArchivedChannels)
|
|
if err != nil {
|
|
return profilePictures, err
|
|
}
|
|
|
|
userLine.User.Teams = members
|
|
|
|
if err := a.exportWriteLine(writer, userLine); err != nil {
|
|
return profilePictures, err
|
|
}
|
|
}
|
|
}
|
|
|
|
return profilePictures, nil
|
|
}
|
|
|
|
func (a *App) exportAllBots(rctx request.CTX, job *model.Job, writer io.Writer, includeProfilePictures bool) ([]string, *model.AppError) {
|
|
afterId := ""
|
|
cnt := 0
|
|
profilePictures := []string{}
|
|
|
|
const pageSize = 1000
|
|
|
|
for {
|
|
bots, err := a.Srv().Store().Bot().GetAllAfter(pageSize, afterId)
|
|
if err != nil {
|
|
return profilePictures, model.NewAppError("exportAllBots", "app.user.get.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
|
|
}
|
|
|
|
cnt += len(bots)
|
|
updateJobProgress(rctx.Logger(), a.Srv().Store(), job, "bots_exported", cnt)
|
|
|
|
for _, bot := range bots {
|
|
afterId = bot.UserId
|
|
|
|
var ownerUsername string
|
|
owner, err := a.Srv().Store().User().Get(rctx.Context(), bot.OwnerId)
|
|
if err != nil {
|
|
var nfErr *store.ErrNotFound
|
|
if errors.As(err, &nfErr) {
|
|
ownerUsername = bot.OwnerId
|
|
} else {
|
|
return profilePictures, model.NewAppError("exportAllBots", "app.user.get.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
|
|
}
|
|
} else {
|
|
ownerUsername = owner.Username
|
|
}
|
|
|
|
botLine := importLineFromBot(bot, ownerUsername)
|
|
|
|
if includeProfilePictures {
|
|
pp, err := a.GetProfileImagePath(model.UserFromBot(bot))
|
|
if err != nil {
|
|
return profilePictures, err
|
|
}
|
|
if pp != "" {
|
|
botLine.Bot.ProfileImage = &pp
|
|
profilePictures = append(profilePictures, pp)
|
|
}
|
|
}
|
|
|
|
if err := a.exportWriteLine(writer, botLine); err != nil {
|
|
return profilePictures, err
|
|
}
|
|
}
|
|
|
|
if len(bots) < pageSize {
|
|
break
|
|
}
|
|
}
|
|
|
|
return profilePictures, nil
|
|
}
|
|
|
|
func (a *App) buildUserTeamAndChannelMemberships(rctx request.CTX, userID string, includeArchivedChannels bool) (*[]imports.UserTeamImportData, *model.AppError) {
|
|
var memberships []imports.UserTeamImportData
|
|
|
|
members, err := a.Srv().Store().Team().GetTeamMembersForExport(userID)
|
|
if err != nil {
|
|
return nil, model.NewAppError("buildUserTeamAndChannelMemberships", "app.team.get_members.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
|
|
}
|
|
|
|
for _, member := range members {
|
|
// Skip deleted.
|
|
if member.DeleteAt != 0 {
|
|
continue
|
|
}
|
|
|
|
memberData := importUserTeamDataFromTeamMember(member)
|
|
|
|
// Do the Channel Memberships.
|
|
channelMembers, err := a.buildUserChannelMemberships(rctx, userID, member.TeamId, includeArchivedChannels)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
// Get the user theme
|
|
themePreference, nErr := a.Srv().Store().Preference().Get(member.UserId, model.PreferenceCategoryTheme, member.TeamId)
|
|
if nErr == nil {
|
|
memberData.Theme = &themePreference.Value
|
|
}
|
|
|
|
memberData.Channels = channelMembers
|
|
|
|
memberships = append(memberships, *memberData)
|
|
}
|
|
|
|
return &memberships, nil
|
|
}
|
|
|
|
func (a *App) buildUserChannelMemberships(rctx request.CTX, userID string, teamID string, includeArchivedChannels bool) (*[]imports.UserChannelImportData, *model.AppError) {
|
|
members, nErr := a.Srv().Store().Channel().GetChannelMembersForExport(userID, teamID, includeArchivedChannels)
|
|
if nErr != nil {
|
|
return nil, model.NewAppError("buildUserChannelMemberships", "app.channel.get_members.app_error", nil, "", http.StatusInternalServerError).Wrap(nErr)
|
|
}
|
|
|
|
category := model.PreferenceCategoryFavoriteChannel
|
|
preferences, err := a.GetPreferenceByCategoryForUser(rctx, userID, category)
|
|
if err != nil && err.StatusCode != http.StatusNotFound {
|
|
return nil, err
|
|
}
|
|
|
|
memberships := make([]imports.UserChannelImportData, len(members))
|
|
for i, member := range members {
|
|
memberships[i] = *importUserChannelDataFromChannelMemberAndPreferences(member, &preferences)
|
|
}
|
|
return &memberships, nil
|
|
}
|
|
|
|
func (a *App) buildUserNotifyProps(notifyProps model.StringMap) *imports.UserNotifyPropsImportData {
|
|
getProp := func(key string) *string {
|
|
if v, ok := notifyProps[key]; ok {
|
|
return &v
|
|
}
|
|
return nil
|
|
}
|
|
|
|
return &imports.UserNotifyPropsImportData{
|
|
Desktop: getProp(model.DesktopNotifyProp),
|
|
DesktopSound: getProp(model.DesktopSoundNotifyProp),
|
|
Email: getProp(model.EmailNotifyProp),
|
|
Mobile: getProp(model.PushNotifyProp),
|
|
MobilePushStatus: getProp(model.PushStatusNotifyProp),
|
|
ChannelTrigger: getProp(model.ChannelMentionsNotifyProp),
|
|
CommentsTrigger: getProp(model.CommentsNotifyProp),
|
|
MentionKeys: getProp(model.MentionKeysNotifyProp),
|
|
}
|
|
}
|
|
|
|
func (a *App) exportAllPosts(rctx request.CTX, job *model.Job, writer io.Writer, withAttachments bool, includeArchivedChannels bool) ([]imports.AttachmentImportData, *model.AppError) {
|
|
var attachments []imports.AttachmentImportData
|
|
afterId := strings.Repeat("0", 26)
|
|
var postProcessCount uint64
|
|
logCheckpoint := time.Now()
|
|
|
|
cnt := 0
|
|
for {
|
|
if time.Since(logCheckpoint) > 5*time.Minute {
|
|
rctx.Logger().Debug(fmt.Sprintf("Bulk Export: processed %d posts", postProcessCount))
|
|
logCheckpoint = time.Now()
|
|
}
|
|
|
|
posts, nErr := a.Srv().Store().Post().GetParentsForExportAfter(1000, afterId, includeArchivedChannels)
|
|
if nErr != nil {
|
|
return nil, model.NewAppError("exportAllPosts", "app.post.get_posts.app_error", nil, "", http.StatusInternalServerError).Wrap(nErr)
|
|
}
|
|
|
|
if len(posts) == 0 {
|
|
return attachments, nil
|
|
}
|
|
cnt += len(posts)
|
|
updateJobProgress(rctx.Logger(), a.Srv().Store(), job, "posts_exported", cnt)
|
|
|
|
for _, post := range posts {
|
|
afterId = post.Id
|
|
postProcessCount++
|
|
|
|
// Skip deleted.
|
|
if post.DeleteAt != 0 {
|
|
continue
|
|
}
|
|
|
|
postLine := importLineForPost(post)
|
|
|
|
replies, replyAttachments, err := a.buildPostReplies(rctx, post.Id, withAttachments)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
followers, err := a.buildThreadFollowers(rctx, post.Id)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
if len(followers) > 0 {
|
|
postLine.Post.ThreadFollowers = &followers
|
|
}
|
|
|
|
if withAttachments && len(replyAttachments) > 0 {
|
|
attachments = append(attachments, replyAttachments...)
|
|
}
|
|
|
|
postLine.Post.Replies = &replies
|
|
postLine.Post.Reactions = &[]imports.ReactionImportData{}
|
|
if post.HasReactions {
|
|
postLine.Post.Reactions, err = a.BuildPostReactions(rctx, post.Id)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
|
|
if len(post.FileIds) > 0 {
|
|
postAttachments, err := a.buildPostAttachments(post.Id)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
postLine.Post.Attachments = &postAttachments
|
|
|
|
if withAttachments && len(postAttachments) > 0 {
|
|
attachments = append(attachments, postAttachments...)
|
|
}
|
|
}
|
|
|
|
if err := a.exportWriteLine(writer, postLine); err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
func (a *App) buildPostReplies(rctx request.CTX, postID string, withAttachments bool) ([]imports.ReplyImportData, []imports.AttachmentImportData, *model.AppError) {
|
|
var replies []imports.ReplyImportData
|
|
var attachments []imports.AttachmentImportData
|
|
|
|
replyPosts, nErr := a.Srv().Store().Post().GetRepliesForExport(postID)
|
|
if nErr != nil {
|
|
return nil, nil, model.NewAppError("buildPostReplies", "app.post.get_posts.app_error", nil, "", http.StatusInternalServerError).Wrap(nErr)
|
|
}
|
|
|
|
for _, reply := range replyPosts {
|
|
replyImportObject := importReplyFromPost(reply)
|
|
if reply.HasReactions {
|
|
var appErr *model.AppError
|
|
replyImportObject.Reactions, appErr = a.BuildPostReactions(rctx, reply.Id)
|
|
if appErr != nil {
|
|
return nil, nil, appErr
|
|
}
|
|
}
|
|
if len(reply.FileIds) > 0 {
|
|
postAttachments, appErr := a.buildPostAttachments(reply.Id)
|
|
if appErr != nil {
|
|
return nil, nil, appErr
|
|
}
|
|
replyImportObject.Attachments = &postAttachments
|
|
if withAttachments && len(postAttachments) > 0 {
|
|
attachments = append(attachments, postAttachments...)
|
|
}
|
|
}
|
|
|
|
replies = append(replies, *replyImportObject)
|
|
}
|
|
|
|
return replies, attachments, nil
|
|
}
|
|
|
|
func (a *App) buildThreadFollowers(_ request.CTX, postID string) ([]imports.ThreadFollowerImportData, *model.AppError) {
|
|
var followers []imports.ThreadFollowerImportData
|
|
|
|
threadFollowers, nErr := a.Srv().Store().Thread().GetThreadMembershipsForExport(postID)
|
|
if nErr != nil {
|
|
return nil, model.NewAppError("buildThreadFollowers", "app.thread.get_threadmembers_for_export.app_error", nil, "", http.StatusInternalServerError).Wrap(nErr)
|
|
}
|
|
|
|
for _, member := range threadFollowers {
|
|
followers = append(followers, *importFollowerFromThreadMember(member))
|
|
}
|
|
|
|
return followers, nil
|
|
}
|
|
|
|
func (a *App) BuildPostReactions(rctx request.CTX, postID string) (*[]ReactionImportData, *model.AppError) {
|
|
var reactionsOfPost []imports.ReactionImportData
|
|
|
|
reactions, nErr := a.Srv().Store().Reaction().GetForPost(postID, true)
|
|
if nErr != nil {
|
|
return nil, model.NewAppError("BuildPostReactions", "app.reaction.get_for_post.app_error", nil, "", http.StatusInternalServerError).Wrap(nErr)
|
|
}
|
|
|
|
for _, reaction := range reactions {
|
|
user, err := a.Srv().Store().User().Get(context.Background(), reaction.UserId)
|
|
if err != nil {
|
|
var nfErr *store.ErrNotFound
|
|
if errors.As(err, &nfErr) { // this is a valid case, the user that reacted might've been deleted by now
|
|
rctx.Logger().Info("Skipping reactions by user since the entity doesn't exist anymore", mlog.String("user_id", reaction.UserId))
|
|
continue
|
|
}
|
|
return nil, model.NewAppError("BuildPostReactions", "app.user.get.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
|
|
}
|
|
reactionsOfPost = append(reactionsOfPost, *importReactionFromPost(user, reaction))
|
|
}
|
|
|
|
return &reactionsOfPost, nil
|
|
}
|
|
|
|
func (a *App) buildPostAttachments(postID string) ([]imports.AttachmentImportData, *model.AppError) {
|
|
infos, nErr := a.Srv().Store().FileInfo().GetForPost(postID, false, false, false)
|
|
if nErr != nil {
|
|
return nil, model.NewAppError("buildPostAttachments", "app.file_info.get_for_post.app_error", nil, "", http.StatusInternalServerError).Wrap(nErr)
|
|
}
|
|
|
|
attachments := make([]imports.AttachmentImportData, 0, len(infos))
|
|
for _, info := range infos {
|
|
attachments = append(attachments, imports.AttachmentImportData{Path: &info.Path})
|
|
}
|
|
|
|
return attachments, nil
|
|
}
|
|
|
|
func (a *App) exportCustomEmoji(rctx request.CTX, job *model.Job, writer io.Writer, outPath, exportDir string, exportFiles bool) ([]string, *model.AppError) {
|
|
var emojiPaths []string
|
|
pageNumber := 0
|
|
cnt := 0
|
|
for {
|
|
customEmojiList, err := a.GetEmojiList(rctx, pageNumber, 100, model.EmojiSortByName)
|
|
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
if len(customEmojiList) == 0 {
|
|
break
|
|
}
|
|
cnt += len(customEmojiList)
|
|
updateJobProgress(rctx.Logger(), a.Srv().Store(), job, "emojis_exported", cnt)
|
|
|
|
pageNumber++
|
|
|
|
emojiPath := filepath.Join(*a.Config().FileSettings.Directory, "emoji")
|
|
pathToDir := filepath.Join(outPath, exportDir)
|
|
if exportFiles {
|
|
if _, err := os.Stat(pathToDir); os.IsNotExist(err) {
|
|
if err := os.Mkdir(pathToDir, os.ModePerm); err != nil {
|
|
return nil, model.NewAppError("BulkExport", "app.export.export_custom_emoji.mkdir.error", nil, "", http.StatusBadRequest).Wrap(err)
|
|
}
|
|
}
|
|
|
|
for _, emoji := range customEmojiList {
|
|
emojiImagePath := filepath.Join(emojiPath, emoji.Id, "image")
|
|
filePath := filepath.Join(exportDir, emoji.Id, "image")
|
|
if exportFiles {
|
|
err := a.copyEmojiImages(rctx, emoji.Id, emojiImagePath, pathToDir)
|
|
if err != nil {
|
|
return nil, model.NewAppError("BulkExport", "app.export.export_custom_emoji.copy_emoji_images.error", nil, "", http.StatusBadRequest).Wrap(err)
|
|
}
|
|
} else {
|
|
filePath = filepath.Join("emoji", emoji.Id, "image")
|
|
emojiPaths = append(emojiPaths, filePath)
|
|
}
|
|
|
|
emojiImportObject := importLineFromEmoji(emoji, filePath)
|
|
if err := a.exportWriteLine(writer, emojiImportObject); err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
}
|
|
}
|
|
return emojiPaths, nil
|
|
}
|
|
|
|
// Copies emoji files from 'data/emoji' dir to 'exported_emoji' dir
|
|
func (a *App) copyEmojiImages(rctx request.CTX, emojiId string, emojiImagePath string, pathToDir string) error {
|
|
fromPath, err := os.Open(emojiImagePath)
|
|
if fromPath == nil || err != nil {
|
|
return errors.New("Error reading " + emojiImagePath + " file")
|
|
}
|
|
defer func() {
|
|
if err = fromPath.Close(); err != nil {
|
|
rctx.Logger().Error("Error closing source file", mlog.String("path", emojiImagePath), mlog.Err(err))
|
|
}
|
|
}()
|
|
|
|
emojiDir := pathToDir + "/" + emojiId
|
|
|
|
if _, err = os.Stat(emojiDir); err != nil {
|
|
if !os.IsNotExist(err) {
|
|
return errors.Wrapf(err, "Error fetching file info of emoji directory %v", emojiDir)
|
|
}
|
|
|
|
if err = os.Mkdir(emojiDir, os.ModePerm); err != nil {
|
|
return errors.Wrapf(err, "Error creating emoji directory %v", emojiDir)
|
|
}
|
|
}
|
|
|
|
toPath, err := os.OpenFile(emojiDir+"/image", os.O_RDWR|os.O_CREATE, 0666)
|
|
if err != nil {
|
|
return errors.New("Error creating the image file " + err.Error())
|
|
}
|
|
defer func() {
|
|
if err = toPath.Close(); err != nil {
|
|
rctx.Logger().Error("Error closing destination file", mlog.String("path", emojiDir+"/image"), mlog.Err(err))
|
|
}
|
|
}()
|
|
_, err = io.Copy(toPath, fromPath)
|
|
if err != nil {
|
|
return errors.New("Error copying emojis " + err.Error())
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func (a *App) exportAllDirectChannels(rctx request.CTX, job *model.Job, writer io.Writer, includeArchivedChannels bool) *model.AppError {
|
|
afterId := strings.Repeat("0", 26)
|
|
cnt := 0
|
|
for {
|
|
channels, err := a.Srv().Store().Channel().GetAllDirectChannelsForExportAfter(1000, afterId, includeArchivedChannels)
|
|
if err != nil {
|
|
return model.NewAppError("exportAllDirectChannels", "app.channel.get_all_direct.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
|
|
}
|
|
|
|
if len(channels) == 0 {
|
|
break
|
|
}
|
|
cnt += len(channels)
|
|
updateJobProgress(rctx.Logger(), a.Srv().Store(), job, "direct_channels_exported", cnt)
|
|
|
|
for _, channel := range channels {
|
|
afterId = channel.Id
|
|
|
|
// Skip deleted.
|
|
if channel.DeleteAt != 0 {
|
|
continue
|
|
}
|
|
|
|
// Skip if there are no active members in the channel
|
|
if len(channel.Members) == 0 {
|
|
continue
|
|
}
|
|
|
|
// Skip if the channel member structure is not intact
|
|
switch channel.Type {
|
|
case model.ChannelTypeGroup:
|
|
groupMembers := make([]string, len(channel.Members))
|
|
for i, m := range channel.Members {
|
|
groupMembers[i] = m.UserId
|
|
}
|
|
if channel.Name != model.GetGroupNameFromUserIds(groupMembers) {
|
|
// this is the case of a group channel when other user is permanently deleted
|
|
// we skip this channel and inform by logging it.
|
|
job.Data["skipped_direct_channels"] = job.Data["skipped_direct_channels"] + "," + channel.Id
|
|
rctx.Logger().Warn("Skipping group channels with partially deleted members", mlog.String("channel_id", channel.Id))
|
|
continue
|
|
}
|
|
case model.ChannelTypeDirect:
|
|
if _, u2 := channel.GetBothUsersForDM(); u2 != "" && len(channel.Members) == 1 {
|
|
// this is the case of a direct channel when other user is permanently deleted
|
|
// we skip this channel and inform by logging it.
|
|
job.Data["skipped_direct_channels"] = job.Data["skipped_direct_channels"] + "," + channel.Id
|
|
rctx.Logger().Info("Skipping direct channel with one active member", mlog.String("channel_id", channel.Id), mlog.String("user_id", channel.Members[0].UserId))
|
|
continue
|
|
}
|
|
}
|
|
|
|
favoritedBy, err := a.buildFavoritedByList(channel.Id)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
shownBy, err := a.buildShownByList(channel)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
channelLine := importLineFromDirectChannel(channel, favoritedBy, shownBy)
|
|
if err := a.exportWriteLine(writer, channelLine); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func (a *App) buildFavoritedByList(channelID string) ([]string, *model.AppError) {
|
|
prefs, err := a.Srv().Store().Preference().GetCategoryAndName(model.PreferenceCategoryFavoriteChannel, channelID)
|
|
if err != nil {
|
|
return nil, model.NewAppError("buildFavoritedByList", "app.preference.get_category.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
|
|
}
|
|
|
|
userIDs := make([]string, 0, len(prefs))
|
|
for _, pref := range prefs {
|
|
if pref.Value != "true" {
|
|
continue
|
|
}
|
|
|
|
user, err := a.Srv().Store().User().Get(context.Background(), pref.UserId)
|
|
if err != nil {
|
|
return nil, model.NewAppError("buildFavoritedByList", "app.user.get.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
|
|
}
|
|
|
|
userIDs = append(userIDs, user.Username)
|
|
}
|
|
|
|
return userIDs, nil
|
|
}
|
|
|
|
func (a *App) buildShownByList(channel *model.DirectChannelForExport) ([]string, *model.AppError) {
|
|
shownBy := make([]string, 0)
|
|
switch channel.Type {
|
|
case model.ChannelTypeGroup:
|
|
for _, member := range channel.Members {
|
|
prefs, err := a.Srv().Store().Preference().GetCategory(member.UserId, model.PreferenceCategoryGroupChannelShow)
|
|
if err != nil {
|
|
return nil, model.NewAppError("buildShownByList", "app.preference.get_category.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
|
|
}
|
|
|
|
for i := range prefs {
|
|
if prefs[i].Name == channel.Id && prefs[i].Value == "true" {
|
|
user, err := a.Srv().Store().User().Get(context.Background(), member.UserId)
|
|
if err != nil {
|
|
return nil, model.NewAppError("buildShownByList", "app.user.get.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
|
|
}
|
|
|
|
shownBy = append(shownBy, user.Username)
|
|
}
|
|
}
|
|
}
|
|
case model.ChannelTypeDirect:
|
|
for i, member := range channel.Members {
|
|
otherMember := member // in case it's a channel with self
|
|
if len(channel.Members) == 2 {
|
|
// since the are only two members, the other member is should be the remainder of i+1/2
|
|
otherMember = channel.Members[(i+1)%2]
|
|
}
|
|
prefs, err := a.Srv().Store().Preference().GetCategoryAndName(model.PreferenceCategoryDirectChannelShow, otherMember.UserId)
|
|
if err != nil {
|
|
return nil, model.NewAppError("buildShownByList", "app.preference.get_category.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
|
|
}
|
|
|
|
for _, pref := range prefs {
|
|
if pref.Value == "true" && pref.UserId == member.UserId {
|
|
user, err := a.Srv().Store().User().Get(context.Background(), member.UserId)
|
|
if err != nil {
|
|
return nil, model.NewAppError("buildShownByList", "app.user.get.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
|
|
}
|
|
|
|
shownBy = append(shownBy, user.Username)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
return shownBy, nil
|
|
}
|
|
|
|
func (a *App) exportAllDirectPosts(rctx request.CTX, job *model.Job, writer io.Writer, withAttachments, includeArchivedChannels bool) ([]imports.AttachmentImportData, *model.AppError) {
|
|
var attachments []imports.AttachmentImportData
|
|
afterId := strings.Repeat("0", 26)
|
|
var postProcessCount uint64
|
|
logCheckpoint := time.Now()
|
|
|
|
cnt := 0
|
|
for {
|
|
if time.Since(logCheckpoint) > 5*time.Minute {
|
|
rctx.Logger().Debug(fmt.Sprintf("Bulk Export: processed %d direct posts", postProcessCount))
|
|
logCheckpoint = time.Now()
|
|
}
|
|
|
|
posts, err := a.Srv().Store().Post().GetDirectPostParentsForExportAfter(1000, afterId, includeArchivedChannels)
|
|
if err != nil {
|
|
return nil, model.NewAppError("exportAllDirectPosts", "app.post.get_direct_posts.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
|
|
}
|
|
|
|
if len(posts) == 0 {
|
|
break
|
|
}
|
|
cnt += len(posts)
|
|
updateJobProgress(rctx.Logger(), a.Srv().Store(), job, "direct_posts_exported", cnt)
|
|
|
|
channelsToSkip := model.SliceToMapKey(strings.Split(job.Data["skipped_direct_channels"], ",")...)
|
|
for _, post := range posts {
|
|
afterId = post.Id
|
|
postProcessCount++
|
|
|
|
// Skip deleted.
|
|
if post.DeleteAt != 0 {
|
|
continue
|
|
}
|
|
|
|
if _, ok := channelsToSkip[post.ChannelId]; ok {
|
|
continue
|
|
}
|
|
|
|
// Handle attachments.
|
|
var postAttachments []imports.AttachmentImportData
|
|
var err *model.AppError
|
|
if len(post.FileIds) > 0 {
|
|
postAttachments, err = a.buildPostAttachments(post.Id)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
if withAttachments && len(postAttachments) > 0 {
|
|
attachments = append(attachments, postAttachments...)
|
|
}
|
|
}
|
|
|
|
// Do the Replies.
|
|
replies, replyAttachments, err := a.buildPostReplies(rctx, post.Id, withAttachments)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
if withAttachments && len(replyAttachments) > 0 {
|
|
attachments = append(attachments, replyAttachments...)
|
|
}
|
|
|
|
postLine := importLineForDirectPost(post)
|
|
postLine.DirectPost.Replies = &replies
|
|
if len(postAttachments) > 0 {
|
|
postLine.DirectPost.Attachments = &postAttachments
|
|
}
|
|
|
|
followers, err := a.buildThreadFollowers(rctx, post.Id)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
if len(followers) > 0 {
|
|
postLine.DirectPost.ThreadFollowers = &followers
|
|
}
|
|
|
|
if err := a.exportWriteLine(writer, postLine); err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
}
|
|
return attachments, nil
|
|
}
|
|
|
|
func (a *App) exportFile(rctx request.CTX, outPath, filePath string, zipWr *zip.Writer) *model.AppError {
|
|
rd, appErr := a.FileReader(filePath)
|
|
if appErr != nil {
|
|
return appErr
|
|
}
|
|
defer func() {
|
|
if err := rd.Close(); err != nil {
|
|
rctx.Logger().Error("Error closing file", mlog.Err(err))
|
|
}
|
|
}()
|
|
|
|
if zipWr != nil {
|
|
wr, err := zipWr.CreateHeader(&zip.FileHeader{
|
|
Name: filepath.Join(model.ExportDataDir, filePath),
|
|
Method: zip.Store,
|
|
})
|
|
if err != nil {
|
|
return model.NewAppError("exportFileAttachment", "app.export.export_attachment.zip_create_header.error",
|
|
nil, "", http.StatusInternalServerError).Wrap(err)
|
|
}
|
|
|
|
if _, err = io.Copy(wr, rd); err != nil {
|
|
return model.NewAppError("exportFileAttachment", "app.export.export_attachment.copy_file.error",
|
|
nil, "", http.StatusInternalServerError).Wrap(err)
|
|
}
|
|
} else {
|
|
filePath = filepath.Join(outPath, model.ExportDataDir, filePath)
|
|
if err := os.MkdirAll(filepath.Dir(filePath), 0700); err != nil {
|
|
return model.NewAppError("exportFileAttachment", "app.export.export_attachment.mkdirall.error",
|
|
nil, "", http.StatusInternalServerError).Wrap(err)
|
|
}
|
|
|
|
file, err := os.Create(filePath)
|
|
if err != nil {
|
|
return model.NewAppError("exportFileAttachment", "app.export.export_attachment.create_file.error",
|
|
nil, "", http.StatusInternalServerError).Wrap(err)
|
|
}
|
|
defer func() {
|
|
if err = file.Close(); err != nil {
|
|
rctx.Logger().Error("Error closing file", mlog.Err(err))
|
|
}
|
|
}()
|
|
|
|
if _, err = io.Copy(file, rd); err != nil {
|
|
return model.NewAppError("exportFileAttachment", "app.export.export_attachment.copy_file.error",
|
|
nil, "", http.StatusInternalServerError).Wrap(err)
|
|
}
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func (a *App) ListExports() ([]string, *model.AppError) {
|
|
exports, appErr := a.ListExportDirectory(*a.Config().ExportSettings.Directory)
|
|
if appErr != nil {
|
|
return nil, appErr
|
|
}
|
|
|
|
results := make([]string, len(exports))
|
|
for i := range exports {
|
|
results[i] = filepath.Base(exports[i])
|
|
}
|
|
|
|
return results, nil
|
|
}
|
|
|
|
func (a *App) GeneratePresignURLForExport(name string) (*model.PresignURLResponse, *model.AppError) {
|
|
if !a.Config().FeatureFlags.EnableExportDirectDownload {
|
|
return nil, model.NewAppError("GeneratePresignURLForExport", "app.eport.generate_presigned_url.featureflag.app_error", nil, "", http.StatusInternalServerError)
|
|
}
|
|
|
|
if !*a.Config().FileSettings.DedicatedExportStore {
|
|
return nil, model.NewAppError("GeneratePresignURLForExport", "app.eport.generate_presigned_url.config.app_error", nil, "", http.StatusInternalServerError)
|
|
}
|
|
|
|
b := a.ExportFileBackend()
|
|
backend, ok := b.(filestore.FileBackendWithLinkGenerator)
|
|
if !ok {
|
|
return nil, model.NewAppError("GeneratePresignURLForExport", "app.eport.generate_presigned_url.driver.app_error", nil, "", http.StatusInternalServerError)
|
|
}
|
|
|
|
p := path.Join(*a.Config().ExportSettings.Directory, filepath.Base(name))
|
|
found, err := b.FileExists(p)
|
|
if err != nil {
|
|
return nil, model.NewAppError("GeneratePresignURLForExport", "app.eport.generate_presigned_url.fileexist.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
|
|
}
|
|
if !found {
|
|
return nil, model.NewAppError("GeneratePresignURLForExport", "app.eport.generate_presigned_url.notfound.app_error", nil, "", http.StatusInternalServerError)
|
|
}
|
|
|
|
link, exp, err := backend.GeneratePublicLink(p)
|
|
if err != nil {
|
|
return nil, model.NewAppError("GeneratePresignURLForExport", "app.eport.generate_presigned_url.link.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
|
|
}
|
|
|
|
return &model.PresignURLResponse{
|
|
URL: link,
|
|
Expiration: exp,
|
|
}, nil
|
|
}
|
|
|
|
func (a *App) DeleteExport(name string) *model.AppError {
|
|
filePath := filepath.Join(*a.Config().ExportSettings.Directory, name)
|
|
|
|
if ok, err := a.ExportFileExists(filePath); err != nil {
|
|
return err
|
|
} else if !ok {
|
|
return nil
|
|
}
|
|
|
|
return a.RemoveExportFile(filePath)
|
|
}
|
|
|
|
func updateJobProgress(logger mlog.LoggerIFace, store store.Store, job *model.Job, key string, value int) {
|
|
if job != nil {
|
|
job.Data[key] = strconv.Itoa(value)
|
|
if _, err2 := store.Job().UpdateOptimistically(job, model.JobStatusInProgress); err2 != nil {
|
|
logger.Warn("Failed to update job status", mlog.Err(err2))
|
|
}
|
|
}
|
|
}
|