From 7886f05e881e45303087a5ae116657f0f6164d81 Mon Sep 17 00:00:00 2001 From: Wim Date: Tue, 20 Feb 2018 00:54:35 +0100 Subject: [PATCH] Download (and upload) avatar images from mattermost and telegram when mediaserver is configured. Closes #362 An extra avatarMap (cache) is created for mattermost and telegram. If MediaServerUpload is configured, the avatar images of users are downloaded the first time a user sends a message. If this download succeeds a message with EVENT_AVATAR_DOWNLOAD is sent to the originating protocol. This message also contains a SHA field (in msg.Extra["file"]), if this is not empty, the sha will be added to the avatarMap. (so we now have a userid-sha cache) Next time this user sends a message, the MediaServerUpload/sha/userid.png URL will be used as the avatar field. --- bridge/config/config.go | 1 + bridge/helper/helper.go | 7 +++++ bridge/mattermost/mattermost.go | 47 +++++++++++++++++++++++++-- bridge/telegram/telegram.go | 56 +++++++++++++++++++++++++++++++-- gateway/gateway.go | 28 ++++++++++++++--- 5 files changed, 129 insertions(+), 10 deletions(-) diff --git a/bridge/config/config.go b/bridge/config/config.go index 1cf0b9f5..c5d4f543 100644 --- a/bridge/config/config.go +++ b/bridge/config/config.go @@ -14,6 +14,7 @@ const ( EVENT_TOPIC_CHANGE = "topic_change" EVENT_FAILURE = "failure" EVENT_FILE_FAILURE_SIZE = "file_failure_size" + EVENT_AVATAR_DOWNLOAD = "avatar_download" EVENT_REJOIN_CHANNELS = "rejoin_channels" EVENT_USER_ACTION = "user_action" EVENT_MSG_DELETE = "msg_delete" diff --git a/bridge/helper/helper.go b/bridge/helper/helper.go index 10c3647a..44052e11 100644 --- a/bridge/helper/helper.go +++ b/bridge/helper/helper.go @@ -54,3 +54,10 @@ func HandleExtra(msg *config.Message, general *config.Protocol) []config.Message } return rmsg } + +func GetAvatar(av map[string]string, userid string, general *config.Protocol) string { + if sha, ok := av[userid]; ok { + return general.MediaServerUpload + "/" + sha + "/" + userid + ".png" + } + return "" +} diff --git a/bridge/mattermost/mattermost.go b/bridge/mattermost/mattermost.go index 1f42b437..e60a136e 100644 --- a/bridge/mattermost/mattermost.go +++ b/bridge/mattermost/mattermost.go @@ -35,6 +35,7 @@ type Bmattermost struct { MMapi TeamId string *config.BridgeConfig + avatarMap map[string]string } var flog *log.Entry @@ -45,7 +46,7 @@ func init() { } func New(cfg *config.BridgeConfig) *Bmattermost { - b := &Bmattermost{BridgeConfig: cfg} + b := &Bmattermost{BridgeConfig: cfg, avatarMap: make(map[string]string)} b.mmMap = make(map[string]string) return b } @@ -149,6 +150,18 @@ func (b *Bmattermost) Send(msg config.Message) (string, error) { message := msg.Text channel := msg.Channel + // map the file SHA to our user (caches the avatar) + if msg.Event == config.EVENT_AVATAR_DOWNLOAD { + fi := msg.Extra["file"][0].(config.FileInfo) + /* if we have a sha we have successfully uploaded the file to the media server, + so we can now cache the sha */ + if fi.SHA != "" { + flog.Debugf("Added %s to %s in avatarMap", fi.SHA, msg.UserID) + b.avatarMap[msg.UserID] = fi.SHA + } + return "", nil + } + if b.Config.PrefixMessagesWithNick { message = nick + message } @@ -235,7 +248,8 @@ func (b *Bmattermost) handleMatter() { go b.handleMatterClient(mchan) } for message := range mchan { - rmsg := config.Message{Username: message.Username, Channel: message.Channel, Account: b.Account, UserID: message.UserID, ID: message.ID, Event: message.Event, Extra: message.Extra} + avatar := helper.GetAvatar(b.avatarMap, message.UserID, b.General) + rmsg := config.Message{Username: message.Username, Channel: message.Channel, Account: b.Account, UserID: message.UserID, ID: message.ID, Event: message.Event, Extra: message.Extra, Avatar: avatar} text, ok := b.replaceAction(message.Text) if ok { rmsg.Event = config.EVENT_USER_ACTION @@ -261,6 +275,11 @@ func (b *Bmattermost) handleMatterClient(mchan chan *MMMessage) { continue } + // only download avatars if we have a place to upload them (configured mediaserver) + if b.General.MediaServerUpload != "" { + b.handleDownloadAvatar(message.UserID, message.Channel) + } + m := &MMMessage{Extra: make(map[string][]interface{})} props := message.Post.Props @@ -365,3 +384,27 @@ func (b *Bmattermost) replaceAction(text string) (string, bool) { } return text, false } + +// handleDownloadAvatar downloads the avatar of userid from channel +// sends a EVENT_AVATAR_DOWNLOAD message to the gateway if successful. +// logs an error message if it fails +func (b *Bmattermost) handleDownloadAvatar(userid string, channel string) { + var name string + msg := config.Message{Username: "system", Text: "avatar", Channel: channel, Account: b.Account, UserID: userid, Event: config.EVENT_AVATAR_DOWNLOAD, Extra: make(map[string][]interface{})} + if _, ok := b.avatarMap[userid]; !ok { + data, resp := b.mc.Client.GetProfileImage(userid, "") + if resp.Error != nil { + flog.Errorf("ProfileImage download failed for %#v %s", userid, resp.Error) + } + if len(data) <= b.General.MediaDownloadSize { + name = userid + ".png" + flog.Debugf("download OK %#v %#v", name, len(data)) + msg.Extra["file"] = append(msg.Extra["file"], config.FileInfo{Name: name, Data: &data, Avatar: true}) + flog.Debugf("Sending avatar download message from %#v on %s to gateway", userid, b.Account) + flog.Debugf("Message is %#v", msg) + b.Remote <- msg + } else { + flog.Errorf("File %#v to large to download (%#v). MediaDownloadSize is %#v", name, len(data), b.General.MediaDownloadSize) + } + } +} diff --git a/bridge/telegram/telegram.go b/bridge/telegram/telegram.go index dfb596aa..ef310ef3 100644 --- a/bridge/telegram/telegram.go +++ b/bridge/telegram/telegram.go @@ -14,6 +14,7 @@ import ( type Btelegram struct { c *tgbotapi.BotAPI *config.BridgeConfig + avatarMap map[string]string // keep cache of userid and avatar sha } var flog *log.Entry @@ -24,7 +25,7 @@ func init() { } func New(cfg *config.BridgeConfig) *Btelegram { - return &Btelegram{BridgeConfig: cfg} + return &Btelegram{BridgeConfig: cfg, avatarMap: make(map[string]string)} } func (b *Btelegram) Connect() error { @@ -63,6 +64,18 @@ func (b *Btelegram) Send(msg config.Message) (string, error) { return "", err } + // map the file SHA to our user (caches the avatar) + if msg.Event == config.EVENT_AVATAR_DOWNLOAD { + fi := msg.Extra["file"][0].(config.FileInfo) + /* if we have a sha we have successfully uploaded the file to the media server, + so we can now cache the sha */ + if fi.SHA != "" { + flog.Debugf("Added %s to %s in avatarMap", fi.SHA, msg.UserID) + b.avatarMap[msg.UserID] = fi.SHA + } + return "", nil + } + if b.Config.MessageFormat == "HTML" { msg.Text = makeHTML(msg.Text) } @@ -173,6 +186,10 @@ func (b *Btelegram) handleRecv(updates <-chan tgbotapi.Update) { } text = message.Text channel = strconv.FormatInt(message.Chat.ID, 10) + // only download avatars if we have a place to upload them (configured mediaserver) + if b.General.MediaServerUpload != "" { + b.handleDownloadAvatar(message.From.ID, channel) + } } if username == "" { @@ -235,8 +252,9 @@ func (b *Btelegram) handleRecv(updates <-chan tgbotapi.Update) { } if text != "" || len(fmsg.Extra) > 0 { + avatar := helper.GetAvatar(b.avatarMap, strconv.Itoa(message.From.ID), b.General) flog.Debugf("Sending message from %s on %s to gateway", username, b.Account) - msg := config.Message{Username: username, Text: text, Channel: channel, Account: b.Account, UserID: strconv.Itoa(message.From.ID), ID: strconv.Itoa(message.MessageID), Extra: fmsg.Extra} + msg := config.Message{Username: username, Text: text, Channel: channel, Account: b.Account, UserID: strconv.Itoa(message.From.ID), ID: strconv.Itoa(message.MessageID), Extra: fmsg.Extra, Avatar: avatar} flog.Debugf("Message is %#v", msg) b.Remote <- msg } @@ -251,6 +269,39 @@ func (b *Btelegram) getFileDirectURL(id string) string { return res } +// handleDownloadAvatar downloads the avatar of userid from channel +// sends a EVENT_AVATAR_DOWNLOAD message to the gateway if successful. +// logs an error message if it fails +func (b *Btelegram) handleDownloadAvatar(userid int, channel string) { + msg := config.Message{Username: "system", Text: "avatar", Channel: channel, Account: b.Account, UserID: strconv.Itoa(userid), Event: config.EVENT_AVATAR_DOWNLOAD, Extra: make(map[string][]interface{})} + if _, ok := b.avatarMap[strconv.Itoa(userid)]; !ok { + photos, err := b.c.GetUserProfilePhotos(tgbotapi.UserProfilePhotosConfig{UserID: userid, Limit: 1}) + if err != nil { + flog.Errorf("Userprofile download failed for %#v %s", userid, err) + } + if len(photos.Photos) > 0 { + photo := photos.Photos[0][0] + url := b.getFileDirectURL(photo.FileID) + name := strconv.Itoa(userid) + ".png" + flog.Debugf("trying to download %#v fileid %#v with size %#v", name, photo.FileID, photo.FileSize) + if photo.FileSize <= b.General.MediaDownloadSize { + data, err := helper.DownloadFile(url) + if err != nil { + flog.Errorf("download %s failed %#v", url, err) + } else { + flog.Debugf("download OK %#v %#v %#v", name, len(*data), len(url)) + msg.Extra["file"] = append(msg.Extra["file"], config.FileInfo{Name: name, Data: data, Avatar: true}) + flog.Debugf("Sending avatar download message from %#v on %s to gateway", userid, b.Account) + flog.Debugf("Message is %#v", msg) + b.Remote <- msg + } + } else { + flog.Errorf("File %#v to large to download (%#v). MediaDownloadSize is %#v", name, photo.FileSize, b.General.MediaDownloadSize) + } + } + } +} + func (b *Btelegram) handleDownload(file interface{}, comment string, msg *config.Message) { size := 0 url := "" @@ -311,7 +362,6 @@ func (b *Btelegram) handleDownload(file interface{}, comment string, msg *config return } // if we have a file attached, download it (in memory) and put a pointer to it in msg.Extra - // limit to 1MB for now flog.Debugf("trying to download %#v fileid %#v with size %#v", name, fileid, size) if size <= b.General.MediaDownloadSize { data, err := helper.DownloadFile(url) diff --git a/gateway/gateway.go b/gateway/gateway.go index c16264a2..6fe7f0ea 100644 --- a/gateway/gateway.go +++ b/gateway/gateway.go @@ -178,6 +178,14 @@ func (gw *Gateway) handleMessage(msg config.Message, dest *bridge.Bridge) []*BrM } } } + + // Avatar downloads are only relevant for telegram and mattermost for now + if msg.Event == config.EVENT_AVATAR_DOWNLOAD { + if dest.Protocol != "mattermost" && + dest.Protocol != "telegram" { + return brMsgIDs + } + } // only relay join/part when configged if msg.Event == config.EVENT_JOIN_LEAVE && !gw.Bridges[dest.Account].Config.ShowJoinPart { return brMsgIDs @@ -185,6 +193,7 @@ func (gw *Gateway) handleMessage(msg config.Message, dest *bridge.Bridge) []*BrM if msg.Event == config.EVENT_TOPIC_CHANGE && !gw.Bridges[dest.Account].Config.ShowTopicChange { return brMsgIDs } + // broadcast to every out channel (irc QUIT) if msg.Channel == "" && msg.Event != config.EVENT_JOIN_LEAVE { log.Debug("empty channel") @@ -194,9 +203,16 @@ func (gw *Gateway) handleMessage(msg config.Message, dest *bridge.Bridge) []*BrM origmsg := msg channels := gw.getDestChannel(&msg, *dest) for _, channel := range channels { - // do not send to ourself - if channel.ID == getChannelID(origmsg) { - continue + // Only send the avatar download event to ourselves. + if msg.Event == config.EVENT_AVATAR_DOWNLOAD { + if channel.ID != getChannelID(origmsg) { + continue + } + } else { + // do not send to ourself for any other event + if channel.ID == getChannelID(origmsg) { + continue + } } log.Debugf("Sending %#v from %s (%s) to %s (%s)", msg, msg.Account, originchannel, dest.Account, channel.Name) msg.Channel = channel.Name @@ -362,15 +378,17 @@ func (gw *Gateway) handleFiles(msg *config.Message) { durl := gw.Config.General.MediaServerDownload + "/" + sha1sum + "/" + fi.Name extra := msg.Extra["file"][i].(config.FileInfo) extra.URL = durl - extra.SHA = sha1sum - msg.Extra["file"][i] = extra req, _ := http.NewRequest("PUT", url, reader) req.Header.Set("Content-Type", "binary/octet-stream") _, err := client.Do(req) if err != nil { log.Errorf("mediaserver upload failed: %#v", err) + continue } log.Debugf("mediaserver download URL = %s", durl) + // we uploaded the file successfully. Add the SHA + extra.SHA = sha1sum + msg.Extra["file"][i] = extra } } }