Release v1.3.0

Refactor message.Extra to a map[string][]interface{} to have a bit more flexibility
for stuffing extra stuff.

For attached files from slack, files < 1MB size get downloaded (in memory), and get
put into Extra["file"][]config.FileInfo (containing a pointer to the buffer and
the filename). This is not async so slack channels with lots of attached files
may suffer a slowdown. (the download timeout is set at 5 seconds).

README.md

@@ -29,9 +29,10 @@ Has a REST API.
 # Features
 * Relays public channel messages between multiple mattermost, IRC, XMPP, Gitter, Slack, Discord, Telegram, Rocket.Chat, Hipchat (via xmpp), Matrix and Steam. 
   Pick and mix.
-* Matterbridge can also work with private groups on your mattermost/slack.
+* Support private groups on your mattermost/slack.
 * Allow for bridging the same bridges, which means you can eg bridge between multiple mattermosts.
 * The bridge is now a gateway which has support multiple in and out bridges. (and supports multiple gateways).
+* Edits and delete messages across bridges that support it (mattermost,slack,discord,gitter,telegram)
 * REST API to read/post messages to bridges (WIP).
 
 # Requirements
@@ -53,7 +54,7 @@ See https://github.com/42wim/matterbridge/wiki
 
 # Installing
 ## Binaries
-* Latest stable release [v1.2.0](https://github.com/42wim/matterbridge/releases/latest)
+* Latest stable release [v1.3.0](https://github.com/42wim/matterbridge/releases/latest)
 * Development releases (follows master) can be downloaded [here](https://dl.bintray.com/42wim/nightly/)  
 
 ## Building

bridge/config/config.go

@@ -29,6 +29,12 @@ type Message struct {
 	Gateway   string    `json:"gateway"`
 	Timestamp time.Time `json:"timestamp"`
 	ID        string    `json:"id"`
+	Extra     map[string][]interface{}
+}
+
+type FileInfo struct {
+	Name string
+	Data *[]byte
 }
 
 type ChannelInfo struct {
@@ -57,6 +63,7 @@ type Protocol struct {
 	Nick                   string // all protocols
 	NickFormatter          string // mattermost, slack
 	NickServNick           string // IRC
+	NickServUsername       string // IRC
 	NickServPassword       string // IRC
 	NicksPerRow            int    // mattermost, slack
 	NoHomeServerSuffix     bool   // matrix

bridge/irc/irc.go

@@ -88,6 +88,7 @@ func (b *Birc) Connect() error {
 		i.Password = b.Config.Password
 	}
 	i.AddCallback(ircm.RPL_WELCOME, b.handleNewConnection)
+	i.AddCallback(ircm.RPL_ENDOFMOTD, b.handleOtherAuth)
 	err := i.Connect(b.Config.Server)
 	if err != nil {
 		return err
@@ -257,6 +258,13 @@ func (b *Birc) handleOther(event *irc.Event) {
 	flog.Debugf("%#v", event.Raw)
 }
 
+func (b *Birc) handleOtherAuth(event *irc.Event) {
+	if strings.EqualFold(b.Config.NickServNick, "Q@CServe.quakenet.org") {
+		flog.Debugf("Authenticating %s against %s", b.Config.NickServUsername, b.Config.NickServNick)
+		b.i.Privmsg(b.Config.NickServNick, "AUTH "+b.Config.NickServUsername+" "+b.Config.NickServPassword)
+	}
+}
+
 func (b *Birc) handlePrivMsg(event *irc.Event) {
 	b.Nick = b.i.GetNick()
 	// freenode doesn't send 001 as first reply

bridge/mattermost/mattermost.go

@@ -26,6 +26,7 @@ type MMMessage struct {
 	UserID   string
 	ID       string
 	Event    string
+	Extra    map[string][]interface{}
 }
 
 type Bmattermost struct {
@@ -162,6 +163,9 @@ func (b *Bmattermost) Send(msg config.Message) (string, error) {
 		matterMessage.UserName = nick
 		matterMessage.Type = ""
 		matterMessage.Text = message
+		matterMessage.Text = message
+		matterMessage.Props = make(map[string]interface{})
+		matterMessage.Props["matterbridge"] = true
 		err := b.mh.Send(matterMessage)
 		if err != nil {
 			flog.Info(err)
@@ -175,6 +179,26 @@ func (b *Bmattermost) Send(msg config.Message) (string, error) {
 		}
 		return msg.ID, b.mc.DeleteMessage(msg.ID)
 	}
+	if msg.Extra != nil {
+		if len(msg.Extra["file"]) > 0 {
+			var err error
+			var res, id string
+			for _, f := range msg.Extra["file"] {
+				fi := f.(config.FileInfo)
+				id, err = b.mc.UploadFile(*fi.Data, b.mc.GetChannelId(channel, ""), fi.Name)
+				if err != nil {
+					flog.Debugf("ERROR %#v", err)
+					return "", err
+				}
+				message = "uploaded a file: " + fi.Name
+				if b.Config.PrefixMessagesWithNick {
+					message = nick + "uploaded a file: " + fi.Name
+				}
+				res, err = b.mc.PostMessageWithFiles(b.mc.GetChannelId(channel, ""), message, []string{id})
+			}
+			return res, err
+		}
+	}
 	if msg.ID != "" {
 		return b.mc.EditMessage(msg.ID, message)
 	}
@@ -195,7 +219,7 @@ func (b *Bmattermost) handleMatter() {
 		go b.handleMatterClient(mchan)
 	}
 	for message := range mchan {
-		rmsg := config.Message{Username: message.Username, Channel: message.Channel, Account: b.Account, UserID: message.UserID, ID: message.ID, Event: message.Event}
+		rmsg := config.Message{Username: message.Username, Channel: message.Channel, Account: b.Account, UserID: message.UserID, ID: message.ID, Event: message.Event, Extra: message.Extra}
 		text, ok := b.replaceAction(message.Text)
 		if ok {
 			rmsg.Event = config.EVENT_USER_ACTION
@@ -220,6 +244,22 @@ func (b *Bmattermost) handleMatterClient(mchan chan *MMMessage) {
 		if (message.Raw.Event == "post_edited") && b.Config.EditDisable {
 			continue
 		}
+
+		m := &MMMessage{Extra: make(map[string][]interface{})}
+
+		props := message.Post.Props
+		if props != nil {
+			if _, ok := props["matterbridge"].(bool); ok {
+				flog.Debugf("sent by matterbridge, ignoring")
+				continue
+			}
+			if _, ok := props["override_username"].(string); ok {
+				message.Username = props["override_username"].(string)
+			}
+			if _, ok := props["attachments"].([]interface{}); ok {
+				m.Extra["attachments"] = props["attachments"].([]interface{})
+			}
+		}
 		// do not post our own messages back to irc
 		// only listen to message from our team
 		if (message.Raw.Event == "posted" || message.Raw.Event == "post_edited" || message.Raw.Event == "post_deleted") &&
@@ -229,7 +269,6 @@ func (b *Bmattermost) handleMatterClient(mchan chan *MMMessage) {
 				continue
 			}
 			flog.Debugf("Receiving from matterclient %#v", message)
-			m := &MMMessage{}
 			m.UserID = message.UserID
 			m.Username = message.Username
 			m.Channel = message.Channel

bridge/slack/slack.go

@@ -1,6 +1,7 @@
 package bslack
 
 import (
+	"bytes"
 	"errors"
 	"fmt"
 	"github.com/42wim/matterbridge/bridge/config"
@@ -8,6 +9,8 @@ import (
 	log "github.com/Sirupsen/logrus"
 	"github.com/nlopes/slack"
 	"html"
+	"io"
+	"net/http"
 	"regexp"
 	"strings"
 	"time"
@@ -163,6 +166,8 @@ func (b *Bslack) Send(msg config.Message) (string, error) {
 		np.IconURL = msg.Avatar
 	}
 	np.Attachments = append(np.Attachments, slack.Attachment{CallbackID: "matterbridge"})
+	np.Attachments = append(np.Attachments, b.createAttach(msg.Extra)...)
+
 	// replace mentions
 	np.LinkNames = 1
 
@@ -248,7 +253,7 @@ func (b *Bslack) handleSlack() {
 		text = b.replaceURL(text)
 		text = html.UnescapeString(text)
 		flog.Debugf("Sending message from %s on %s to gateway", message.Username, b.Account)
-		msg := config.Message{Text: text, Username: message.Username, Channel: message.Channel, Account: b.Account, Avatar: b.getAvatar(message.Username), UserID: message.UserID, ID: "slack " + message.Raw.Timestamp}
+		msg := config.Message{Text: text, Username: message.Username, Channel: message.Channel, Account: b.Account, Avatar: b.getAvatar(message.Username), UserID: message.UserID, ID: "slack " + message.Raw.Timestamp, Extra: make(map[string][]interface{})}
 		if message.Raw.SubType == "me_message" {
 			msg.Event = config.EVENT_USER_ACTION
 		}
@@ -265,6 +270,19 @@ func (b *Bslack) handleSlack() {
 			msg.Event = config.EVENT_MSG_DELETE
 			msg.ID = "slack " + message.Raw.DeletedTimestamp
 		}
+
+		// if we have a file attached, download it (in memory) and put a pointer to it in msg.Extra
+		if message.Raw.File != nil {
+			// limit to 1MB for now
+			if message.Raw.File.Size <= 1000000 {
+				data, err := b.downloadFile(message.Raw.File.URLPrivateDownload)
+				if err != nil {
+					flog.Errorf("download %s failed %#v", message.Raw.File.URLPrivateDownload, err)
+				} else {
+					msg.Extra["file"] = append(msg.Extra["file"], config.FileInfo{Name: message.Raw.File.Name, Data: data})
+				}
+			}
+		}
 		flog.Debugf("Message is %#v", msg)
 		b.Remote <- msg
 	}
@@ -285,6 +303,12 @@ func (b *Bslack) handleSlackClient(mchan chan *MMMessage) {
 				flog.Debugf("SubMessage %#v", ev.SubMessage)
 				ev.User = ev.SubMessage.User
 				ev.Text = ev.SubMessage.Text + b.Config.EditSuffix
+
+				// it seems ev.SubMessage.Edited == nil when slack unfurls
+				// do not forward these messages #266
+				if ev.SubMessage.Edited == nil {
+					continue
+				}
 			}
 			// use our own func because rtm.GetChannelInfo doesn't work for private channels
 			channel, err := b.getChannelByID(ev.Channel)
@@ -389,3 +413,47 @@ func (b *Bslack) replaceURL(text string) string {
 	}
 	return text
 }
+
+func (b *Bslack) createAttach(extra map[string][]interface{}) []slack.Attachment {
+	var attachs []slack.Attachment
+	for _, v := range extra["attachments"] {
+		entry := v.(map[string]interface{})
+		s := slack.Attachment{}
+		s.Fallback = entry["fallback"].(string)
+		s.Color = entry["color"].(string)
+		s.Pretext = entry["pretext"].(string)
+		s.AuthorName = entry["author_name"].(string)
+		s.AuthorLink = entry["author_link"].(string)
+		s.AuthorIcon = entry["author_icon"].(string)
+		s.Title = entry["title"].(string)
+		s.TitleLink = entry["title_link"].(string)
+		s.Text = entry["text"].(string)
+		s.ImageURL = entry["image_url"].(string)
+		s.ThumbURL = entry["thumb_url"].(string)
+		s.Footer = entry["footer"].(string)
+		s.FooterIcon = entry["footer_icon"].(string)
+		attachs = append(attachs, s)
+	}
+	return attachs
+}
+
+func (b *Bslack) downloadFile(url string) (*[]byte, error) {
+	var buf bytes.Buffer
+	client := &http.Client{
+		Timeout: time.Second * 5,
+	}
+	req, err := http.NewRequest("GET", url, nil)
+	if err != nil {
+		return nil, err
+	}
+	req.Header.Add("Authorization", "Bearer "+b.Config.Token)
+	resp, err := client.Do(req)
+	if err != nil {
+		resp.Body.Close()
+		return nil, err
+	}
+	io.Copy(&buf, resp.Body)
+	data := buf.Bytes()
+	resp.Body.Close()
+	return &data, nil
+}

bridge/telegram/telegram.go

@@ -160,9 +160,32 @@ func (b *Btelegram) handleRecv(updates <-chan tgbotapi.Update) {
 		if message.Document != nil && b.Config.UseInsecureURL {
 			text = text + " " + message.Document.FileName + " : " + b.getFileDirectURL(message.Document.FileID)
 		}
+
+		// quote the previous message
+		if message.ReplyToMessage != nil {
+			usernameReply := ""
+			if message.ReplyToMessage.From != nil {
+				if b.Config.UseFirstName {
+					usernameReply = message.ReplyToMessage.From.FirstName
+				}
+				if usernameReply == "" {
+					usernameReply = message.ReplyToMessage.From.UserName
+					if usernameReply == "" {
+						usernameReply = message.ReplyToMessage.From.FirstName
+					}
+				}
+			}
+			if usernameReply == "" {
+				usernameReply = "unknown"
+			}
+			text = text + " (re @" + usernameReply + ":" + message.ReplyToMessage.Text + ")"
+		}
+
 		if text != "" {
 			flog.Debugf("Sending message from %s on %s to gateway", username, b.Account)
-			b.Remote <- config.Message{Username: username, Text: text, Channel: channel, Account: b.Account, UserID: strconv.Itoa(message.From.ID), ID: strconv.Itoa(message.MessageID)}
+			msg := config.Message{Username: username, Text: text, Channel: channel, Account: b.Account, UserID: strconv.Itoa(message.From.ID), ID: strconv.Itoa(message.MessageID)}
+			flog.Debugf("Message is %#v", msg)
+			b.Remote <- msg
 		}
 	}
 }

changelog.md

@@ -1,3 +1,17 @@
+# v1.3.0
+## New features
+* Relay slack_attachments from mattermost to slack (slack). Closes #260
+* Add support for quoting previous message when replying (telegram). #237
+* Add support for Quakenet auth (irc). Closes #263
+* Download files (max size 1MB) from slack and reupload to mattermost (slack/mattermost). Closes #255
+
+## Enhancements
+* Backoff for 60 seconds when reconnecting too fast (irc) #267
+* Use override username if specified (mattermost). #260
+
+## Bugfix
+* Try to not forward slack unfurls. Closes #266
+
 # v1.2.0
 ## Breaking changes
 * If you're running a discord bridge, update to this release before 16 october otherwise

gateway/gateway.go

@@ -147,6 +147,17 @@ func (gw *Gateway) getDestChannel(msg *config.Message, dest bridge.Bridge) []con
 
 func (gw *Gateway) handleMessage(msg config.Message, dest *bridge.Bridge) []*BrMsgID {
 	var brMsgIDs []*BrMsgID
+
+	// TODO refactor
+	// only slack now, check will have to be done in the different bridges.
+	// we need to check if we can't use fallback or text in other bridges
+	if msg.Extra != nil {
+		if dest.Protocol != "slack" {
+			if msg.Text == "" {
+				return brMsgIDs
+			}
+		}
+	}
 	// only relay join/part when configged
 	if msg.Event == config.EVENT_JOIN_LEAVE && !gw.Bridges[dest.Account].Config.ShowJoinPart {
 		return brMsgIDs
@@ -199,6 +210,10 @@ func (gw *Gateway) ignoreMessage(msg *config.Message) bool {
 		return true
 	}
 	if msg.Text == "" {
+		// we have an attachment
+		if msg.Extra != nil && msg.Extra["attachments"] != nil {
+			return false
+		}
 		log.Debugf("ignoring empty message %#v from %s", msg, msg.Account)
 		return true
 	}

matterbridge.go

@@ -11,7 +11,7 @@ import (
 )
 
 var (
-	version = "1.2.0"
+	version = "1.3.0"
 	githash string
 )
 

matterbridge.toml.sample

@@ -55,10 +55,15 @@ Nick="matterbot"
 
 #If you registered your bot with a service like Nickserv on freenode. 
 #Also being used when UseSASL=true
+#
+#Note: if you want do to quakenet auth, set NickServNick="Q@CServe.quakenet.org"
 #OPTIONAL
 NickServNick="nickserv"
 NickServPassword="secret"
 
+#OPTIONAL only used for quakenet auth
+NickServUsername="username"
+
 #Flood control
 #Delay in milliseconds between each message send to the IRC server
 #OPTIONAL (default 1300)

matterclient/matterclient.go

@@ -281,7 +281,7 @@ func (m *MMClient) WsReceiver() {
 			}
 			// if we have file attached but the message is empty, also send it
 			if msg.Post != nil {
-				if msg.Text != "" || len(msg.Post.FileIds) > 0 {
+				if msg.Text != "" || len(msg.Post.FileIds) > 0 || msg.Post.Type == "slack_attachment" {
 					m.MessageChan <- msg
 				}
 			}
@@ -467,6 +467,15 @@ func (m *MMClient) PostMessage(channelId string, text string) (string, error) {
 	return res.Id, nil
 }
 
+func (m *MMClient) PostMessageWithFiles(channelId string, text string, fileIds []string) (string, error) {
+	post := &model.Post{ChannelId: channelId, Message: text, FileIds: fileIds}
+	res, resp := m.Client.CreatePost(post)
+	if resp.Error != nil {
+		return "", resp.Error
+	}
+	return res.Id, nil
+}
+
 func (m *MMClient) EditMessage(postId string, text string) (string, error) {
 	post := &model.Post{Message: text}
 	res, resp := m.Client.UpdatePost(postId, post)
@@ -780,6 +789,14 @@ func (m *MMClient) GetTeamId() string {
 	return m.Team.Id
 }
 
+func (m *MMClient) UploadFile(data []byte, channelId string, filename string) (string, error) {
+	f, resp := m.Client.UploadFile(data, channelId, filename)
+	if resp.Error != nil {
+		return "", resp.Error
+	}
+	return f.FileInfos[0].Id, nil
+}
+
 func (m *MMClient) StatusLoop() {
 	retries := 0
 	backoff := time.Second * 60

matterhook/matterhook.go

@@ -17,13 +17,14 @@ import (
 
 // OMessage for mattermost incoming webhook. (send to mattermost)
 type OMessage struct {
-	Channel     string      `json:"channel,omitempty"`
-	IconURL     string      `json:"icon_url,omitempty"`
-	IconEmoji   string      `json:"icon_emoji,omitempty"`
-	UserName    string      `json:"username,omitempty"`
-	Text        string      `json:"text"`
-	Attachments interface{} `json:"attachments,omitempty"`
-	Type        string      `json:"type,omitempty"`
+	Channel     string                 `json:"channel,omitempty"`
+	IconURL     string                 `json:"icon_url,omitempty"`
+	IconEmoji   string                 `json:"icon_emoji,omitempty"`
+	UserName    string                 `json:"username,omitempty"`
+	Text        string                 `json:"text"`
+	Attachments interface{}            `json:"attachments,omitempty"`
+	Type        string                 `json:"type,omitempty"`
+	Props       map[string]interface{} `json:"props"`
 }
 
 // IMessage for mattermost outgoing webhook. (received from mattermost)

vendor/github.com/42wim/go-ircevent/irc.go

@@ -227,12 +227,17 @@ func (irc *Connection) isQuitting() bool {
 // Main loop to control the connection.
 func (irc *Connection) Loop() {
 	errChan := irc.ErrorChan()
+	connTime := time.Now()
 	for !irc.isQuitting() {
 		err := <-errChan
 		close(irc.end)
 		irc.Wait()
 		for !irc.isQuitting() {
 			irc.Log.Printf("Error, disconnected: %s\n", err)
+			if time.Now().Sub(connTime) < time.Second*5 {
+				irc.Log.Println("Rreconnecting too fast, sleeping 60 seconds")
+				time.Sleep(60 * time.Second)
+			}
 			if err = irc.Reconnect(); err != nil {
 				irc.Log.Printf("Error while reconnecting: %s\n", err)
 				time.Sleep(60 * time.Second)
@@ -241,6 +246,7 @@ func (irc *Connection) Loop() {
 				break
 			}
 		}
+		connTime = time.Now()
 	}
 }
 

vendor/manifest

@@ -13,7 +13,7 @@
 			"importpath": "github.com/42wim/go-ircevent",
 			"repository": "https://github.com/42wim/go-ircevent",
 			"vcs": "git",
-			"revision": "d3aec637ae2f2a4f9ff95df55091894d80fa3112",
+			"revision": "469ee24527988eda3f400a017cb3276104ea0803",
 			"branch": "ircv3",
 			"notests": true
 		},