From 436a0098dc972cad8a3c60484fd8bf2ea81d9dae Mon Sep 17 00:00:00 2001 From: George Goldberg Date: Sun, 14 Jan 2024 21:10:50 +0000 Subject: [PATCH] Add additional metadata to index. (#29) This includes: * Team name, id, display name * Channel name, id, display name and type * Basic LegalHold details (name, id, display name, start time, last executed at time) Also adds TeamID to CSV files of messages. This is to make the HTML processed version more human-friendly. Fixes #22 --- plugin.json | 2 +- server/legalhold/legal_hold.go | 56 +++++++++++++-- server/model/channel.go | 11 +++ server/model/export.go | 1 + server/model/index.go | 104 +++++++++++++++++++++++++--- server/model/index_test.go | 6 +- server/store/sqlstore/legal_hold.go | 44 ++++++++++++ 7 files changed, 205 insertions(+), 19 deletions(-) create mode 100644 server/model/channel.go diff --git a/plugin.json b/plugin.json index 4f5eb9f..cbf2592 100644 --- a/plugin.json +++ b/plugin.json @@ -6,7 +6,7 @@ "support_url": "https://github.com/mattermost/mattermost-plugin-legal-hold/issues", "release_notes_url": "https://github.com/mattermost/mattermost-plugin-legal-hold/releases/tag/v0.1.0", "icon_path": "assets/starter-template-icon.svg", - "version": "0.1.0", + "version": "0.2.0", "min_server_version": "6.2.1", "server": { "executables": { diff --git a/server/legalhold/legal_hold.go b/server/legalhold/legal_hold.go index 1d4ed4c..f9177fa 100644 --- a/server/legalhold/legal_hold.go +++ b/server/legalhold/legal_hold.go @@ -43,7 +43,7 @@ func NewExecution(legalHold model.LegalHold, papi plugin.API, store *sqlstore.SQ ExecutionEndTime: legalHold.NextExecutionEndTime(), store: store, fileBackend: fileBackend, - index: make(model.LegalHoldIndex), + index: model.NewLegalHoldIndex(), papi: papi, } } @@ -86,8 +86,8 @@ func (ex *Execution) GetChannels() error { // Add to channels index for _, channelID := range channelIDs { - if idx, ok := ex.index[userID]; !ok { - ex.index[userID] = model.LegalHoldIndexUser{ + if idx, ok := ex.index.Users[userID]; !ok { + ex.index.Users[userID] = model.LegalHoldIndexUser{ Username: user.Username, Email: user.Email, Channels: []model.LegalHoldChannelMembership{ @@ -99,7 +99,7 @@ func (ex *Execution) GetChannels() error { }, } } else { - ex.index[userID] = model.LegalHoldIndexUser{ + ex.index.Users[userID] = model.LegalHoldIndexUser{ Username: user.Username, Email: user.Email, Channels: append(idx.Channels, model.LegalHoldChannelMembership{ @@ -216,7 +216,53 @@ func (ex *Execution) ExportFiles(channelID string, batchCreateAt int64, batchPos func (ex *Execution) UpdateIndexes() error { filePath := ex.indexPath() - // Check if the channels index already exists in the file backend. + // Populate the metadata in the index. + ex.index.LegalHold.ID = ex.LegalHold.ID + ex.index.LegalHold.DisplayName = ex.LegalHold.DisplayName + ex.index.LegalHold.Name = ex.LegalHold.Name + ex.index.LegalHold.StartsAt = ex.LegalHold.StartsAt + ex.index.LegalHold.LastExecutionEndedAt = ex.ExecutionEndTime + + if len(ex.channelIDs) > 0 { + metadata, err := ex.store.GetChannelMetadataForIDs(ex.channelIDs) + if err != nil { + return err + } + + for _, m := range metadata { + foundTeam := false + for _, t := range ex.index.Teams { + if t.ID == m.TeamID { + foundTeam = true + t.Channels = append(t.Channels, &model.LegalHoldChannel{ + ID: m.ChannelID, + Name: m.ChannelName, + DisplayName: m.ChannelDisplayName, + Type: m.ChannelType, + }) + break + } + } + + if !foundTeam { + ex.index.Teams = append(ex.index.Teams, &model.LegalHoldTeam{ + ID: m.TeamID, + Name: m.TeamName, + DisplayName: m.TeamDisplayName, + Channels: []*model.LegalHoldChannel{ + { + ID: m.ChannelID, + Name: m.ChannelName, + DisplayName: m.ChannelDisplayName, + Type: m.ChannelType, + }, + }, + }) + } + } + } + + // Check if the index already exists in the file backend. if exists, err := ex.fileBackend.FileExists(filePath); err != nil { return err } else if exists { diff --git a/server/model/channel.go b/server/model/channel.go new file mode 100644 index 0000000..21c156e --- /dev/null +++ b/server/model/channel.go @@ -0,0 +1,11 @@ +package model + +type ChannelMetadata struct { + ChannelID string + ChannelName string + ChannelDisplayName string + ChannelType string + TeamID string + TeamName string + TeamDisplayName string +} diff --git a/server/model/export.go b/server/model/export.go index d652c58..10dfe93 100644 --- a/server/model/export.go +++ b/server/model/export.go @@ -25,6 +25,7 @@ func NewLegalHoldCursor(startTime int64) LegalHoldCursor { type LegalHoldPost struct { // From Team + TeamID string `csv:"TeamId"` TeamName string `csv:"TeamName"` TeamDisplayName string `csv:"TeamDisplayName"` diff --git a/server/model/index.go b/server/model/index.go index 52045f9..93a1264 100644 --- a/server/model/index.go +++ b/server/model/index.go @@ -3,27 +3,111 @@ package model import "github.com/mattermost/mattermost-plugin-legal-hold/server/utils" // LegalHoldChannelMembership represents the membership of a channel by a user in the -// LegalHoldIndex. +// LegalHoldIndexUsers. type LegalHoldChannelMembership struct { - ChannelID string - StartTime int64 - EndTime int64 + ChannelID string `json:"channel_id"` + StartTime int64 `json:"start_time"` + EndTime int64 `json:"end_time"` } -// LegalHoldIndexUser represents the data about one user in the LegalHoldIndex. +// LegalHoldIndexUser represents the data about one user in the LegalHoldIndexUsers. type LegalHoldIndexUser struct { - Username string - Email string - Channels []LegalHoldChannelMembership + Username string `json:"username"` + Email string `json:"email"` + Channels []LegalHoldChannelMembership `json:"channels"` } -// LegalHoldIndex maps to the contents of the index.json file in a legal hold export. +// LegalHoldIndexUsers maps to the contents of the index.json file in a legal hold export. // It contains various pieces of metadata to help with the programmatic and manual processing of // the legal hold export. -type LegalHoldIndex map[string]LegalHoldIndexUser +type LegalHoldIndexUsers map[string]LegalHoldIndexUser + +type LegalHoldIndexDetails struct { + ID string `json:"id"` + Name string `json:"name"` + DisplayName string `json:"display_name"` + StartsAt int64 `json:"starts_at"` + LastExecutionEndedAt int64 `json:"last_execution_ended_at"` +} + +type LegalHoldIndex struct { + Users LegalHoldIndexUsers `json:"users"` + LegalHold LegalHoldIndexDetails `json:"legal_hold"` + Teams []*LegalHoldTeam `json:"teams"` +} + +type LegalHoldTeam struct { + ID string `json:"id"` + Name string `json:"name"` + DisplayName string `json:"display_name"` + Channels []*LegalHoldChannel `json:"channels"` +} + +type LegalHoldChannel struct { + ID string `json:"id"` + Name string `json:"name"` + DisplayName string `json:"display_name"` + Type string `json:"type"` +} + +func NewLegalHoldIndex() LegalHoldIndex { + return LegalHoldIndex{ + Users: make(LegalHoldIndexUsers), + LegalHold: LegalHoldIndexDetails{}, + Teams: make([]*LegalHoldTeam, 0), + } +} // Merge merges the new LegalHoldIndex into this LegalHoldIndex. func (lhi *LegalHoldIndex) Merge(new *LegalHoldIndex) { + // To merge the LegalHold data we overwrite the old struct in full + // with the new one. + lhi.LegalHold = new.LegalHold + + // Recursively merge the Teams (and their Channels) property, taking + // the newest version for the union of both lists. + for _, newTeam := range new.Teams { + found := false + for _, oldTeam := range lhi.Teams { + if newTeam.ID == oldTeam.ID { + oldTeam.Merge(newTeam) + found = true + break + } + } + + if !found { + lhi.Teams = append(lhi.Teams, newTeam) + } + } + + lhi.Users.Merge(&new.Users) +} + +// Merge merges the new LegalHoldTeam into this LegalHoldTeam. +func (team *LegalHoldTeam) Merge(new *LegalHoldTeam) { + team.Name = new.Name + team.DisplayName = new.DisplayName + + for _, newChannel := range new.Channels { + found := false + for _, oldChannel := range team.Channels { + if newChannel.ID == oldChannel.ID { + oldChannel.Name = newChannel.Name + oldChannel.DisplayName = newChannel.DisplayName + found = true + break + } + } + + if !found { + team.Channels = append(team.Channels, newChannel) + } + } +} + +// Merge merges the new LegalHoldIndexUsers into this LegalHoldIndexUsers. +func (lhi *LegalHoldIndexUsers) Merge(new *LegalHoldIndexUsers) { for userID, newUser := range *new { if oldUser, ok := (*lhi)[userID]; !ok { (*lhi)[userID] = newUser diff --git a/server/model/index_test.go b/server/model/index_test.go index 927a443..42ddeb9 100644 --- a/server/model/index_test.go +++ b/server/model/index_test.go @@ -8,7 +8,7 @@ import ( ) func TestMerge(t *testing.T) { - oldIndex := LegalHoldIndex{ + oldIndex := LegalHoldIndexUsers{ "user1": { Username: "oldUser", Email: "oldUser@example.com", @@ -27,7 +27,7 @@ func TestMerge(t *testing.T) { }, } - newIndex := LegalHoldIndex{ + newIndex := LegalHoldIndexUsers{ "user1": { Username: "newUser", Email: "newUser@example.com", @@ -46,7 +46,7 @@ func TestMerge(t *testing.T) { }, } - expectedIndexAfterMerge := LegalHoldIndex{ + expectedIndexAfterMerge := LegalHoldIndexUsers{ "user1": { Username: "newUser", Email: "newUser@example.com", diff --git a/server/store/sqlstore/legal_hold.go b/server/store/sqlstore/legal_hold.go index fcce2b0..1c14fbf 100644 --- a/server/store/sqlstore/legal_hold.go +++ b/server/store/sqlstore/legal_hold.go @@ -2,6 +2,7 @@ package sqlstore import ( sq "github.com/Masterminds/squirrel" + "github.com/jmoiron/sqlx" "github.com/pkg/errors" "github.com/mattermost/mattermost-plugin-legal-hold/server/model" @@ -21,6 +22,7 @@ func (ss SQLStore) GetPostsBatch(channelID string, endTime int64, cursor model.L query := ` SELECT + COALESCE(Teams.Id, '00000000000000000000000000') AS TeamID, COALESCE(Teams.Name, 'direct-messages') AS TeamName, COALESCE(Teams.DisplayName, 'Direct Messages') AS TeamDisplayName, Channels.Name AS ChannelName, @@ -138,3 +140,45 @@ func (ss SQLStore) GetFileInfosByIDs(ids []string) ([]model.FileInfo, error) { return fileInfos, nil } + +func (ss SQLStore) GetChannelMetadataForIDs(channelIDs []string) ([]model.ChannelMetadata, error) { + var data []model.ChannelMetadata + + query := ` + SELECT + COALESCE(Teams.Id, '00000000000000000000000000') AS TeamID, + COALESCE(Teams.Name, 'direct-messages') AS TeamName, + COALESCE(Teams.DisplayName, 'Direct Messages') AS TeamDisplayName, + Channels.Id as ChannelID, + Channels.Name AS ChannelName, + Channels.Type AS ChannelType, + CASE + WHEN Channels.Type = 'D' THEN + ( + (select Users.Username from Users where Users.Id = split_part(Channels.Name, '__', 1)) + || ', ' || + (select Users.Username from Users where Users.Id = split_part(Channels.Name, '__', 2)) + ) + ELSE + Channels.DisplayName + END + AS ChannelDisplayName + FROM + Channels + LEFT OUTER JOIN + Teams ON Teams.ID = Channels.TeamId + WHERE + Channels.Id IN (?)` + + query, args, err := sqlx.In(query, channelIDs) + if err != nil { + return nil, errors.Wrap(err, "unable to get channel metadata") + } + query = ss.replica.Rebind(query) + + if err := ss.replica.Select(&data, query, args...); err != nil { + return nil, errors.Wrap(err, "unable to get channel metadata") + } + + return data, nil +}