Add initial transmitter implementation (discord)

This has been tested with one webhook in one channel.

Sends, edits and deletions work fine

bridge/discord/discord.go

@@ -8,6 +8,7 @@ import (
 
 	"github.com/42wim/matterbridge/bridge"
 	"github.com/42wim/matterbridge/bridge/config"
+	"github.com/42wim/matterbridge/bridge/discord/transmitter"
 	"github.com/42wim/matterbridge/bridge/helper"
 	"github.com/matterbridge/discordgo"
 )
@@ -22,9 +23,6 @@ type Bdiscord struct {
 	nick    string
 	userID  string
 	guildID string
-	webhookID       string
-	webhookToken    string
-	canEditWebhooks bool
 
 	channelsMutex  sync.RWMutex
 	channels       []*discordgo.Channel
@@ -33,6 +31,10 @@ type Bdiscord struct {
 	membersMutex  sync.RWMutex
 	userMemberMap map[string]*discordgo.Member
 	nickMemberMap map[string]*discordgo.Member
+
+	// Webhook specific logic
+	useAutoWebhooks bool
+	transmitter     *transmitter.Transmitter
 }
 
 func New(cfg *bridge.Config) bridge.Bridger {
@@ -40,9 +42,17 @@ func New(cfg *bridge.Config) bridge.Bridger {
 	b.userMemberMap = make(map[string]*discordgo.Member)
 	b.nickMemberMap = make(map[string]*discordgo.Member)
 	b.channelInfoMap = make(map[string]*config.ChannelInfo)
-	if b.GetString("WebhookURL") != "" {
+
-		b.Log.Debug("Configuring Discord Incoming Webhook")
+	// If WebhookURL is set to anything, we assume preference for autoWebhooks
-		b.webhookID, b.webhookToken = b.splitURL(b.GetString("WebhookURL"))
+	//
+	// Legacy note: WebhookURL used to have an actual webhook URL that we would edit,
+	// but we stopped doing that due to Discord making rate limits more aggressive.
+	//
+	// We're keeping the same setting for now, and we will late deprecate this setting
+	// in favour of a new setting, something like "AutoWebhooks=true"
+	b.useAutoWebhooks = b.GetString("WebhookURL") != ""
+	if b.useAutoWebhooks {
+		b.Log.Debug("Using automatic webhooks")
 	}
 	return b
 }
@@ -137,36 +147,44 @@ func (b *Bdiscord) Connect() error {
 		return err
 	}
 
-	b.channelsMutex.RLock()
+	// Initialise webhook management
-	if b.GetString("WebhookURL") == "" {
+	b.transmitter = transmitter.New(b.c, b.guildID, "matterbridge", b.useAutoWebhooks)
-		for _, channel := range b.channels {
+	b.transmitter.Log = b.Log
-			b.Log.Debugf("found channel %#v", channel)
-		}
-	} else {
-		manageWebhooks := discordgo.PermissionManageWebhooks
-		var channelsDenied []string
-		for _, info := range b.Channels {
-			id := b.getChannelID(info.Name) // note(qaisjp): this readlocks channelsMutex
-			b.Log.Debugf("Verifying PermissionManageWebhooks for %s with ID %s", info.ID, id)
 
-			perms, permsErr := b.c.UserChannelPermissions(userinfo.ID, id)
+	var webhookChannelIDs []string
-			if permsErr != nil {
+	for _, channel := range b.Channels {
-				b.Log.Warnf("Failed to check PermissionManageWebhooks in channel \"%s\": %s", info.Name, permsErr.Error())
+		channelID := b.getChannelID(channel.Name) // note(qaisjp): this readlocks channelsMutex
-			} else if perms&manageWebhooks == manageWebhooks {
+
+		// If a WebhookURL was not explicitly provided for this channel,
+		// there are two options: just a regular bot message (ugly) or this is should be webhook sent
+		if channel.Options.WebhookURL == "" {
+			// If it should be webhook sent, we should enforce this via the transmitter
+			if b.useAutoWebhooks {
+				webhookChannelIDs = append(webhookChannelIDs, channelID)
+			}
 			continue
 		}
-			channelsDenied = append(channelsDenied, fmt.Sprintf("%#v", info.Name))
+
+		whID, whToken, ok := b.splitURL(channel.Options.WebhookURL)
+		if !ok {
+			return fmt.Errorf("failed to parse WebhookURL %#v for channel %#v", channel.Options.WebhookURL, channel.ID)
 		}
 
-		b.canEditWebhooks = len(channelsDenied) == 0
+		b.transmitter.AddWebhook(channelID, &discordgo.Webhook{
-		if b.canEditWebhooks {
+			ID:        whID,
-			b.Log.Info("Can manage webhooks; will edit channel for global webhook on send")
+			Token:     whToken,
-		} else {
+			GuildID:   b.guildID,
-			b.Log.Warn("Can't manage webhooks; won't edit channel for global webhook on send")
+			ChannelID: channelID,
-			b.Log.Warn("Can't manage webhooks in channels: ", strings.Join(channelsDenied, ", "))
+		})
+	}
+
+	if b.useAutoWebhooks {
+		err = b.transmitter.RefreshGuildWebhooks(webhookChannelIDs)
+		if err != nil {
+			b.Log.WithError(err).Println("transmitter could not refresh guild webhooks")
+			return err
 		}
 	}
-	b.channelsMutex.RUnlock()
 
 	// Obtaining guild members and initializing nickname mapping.
 	b.membersMutex.Lock()
@@ -223,23 +241,9 @@ func (b *Bdiscord) Send(msg config.Message) (string, error) {
 		msg.Text = "_" + msg.Text + "_"
 	}
 
-	// use initial webhook configured for the entire Discord account
-	isGlobalWebhook := true
-	wID := b.webhookID
-	wToken := b.webhookToken
-
-	// check if have a channel specific webhook
-	b.channelsMutex.RLock()
-	if ci, ok := b.channelInfoMap[msg.Channel+b.Account]; ok {
-		if ci.Options.WebhookURL != "" {
-			wID, wToken = b.splitURL(ci.Options.WebhookURL)
-			isGlobalWebhook = false
-		}
-	}
-	b.channelsMutex.RUnlock()
-
 	// Use webhook to send the message
-	if wID != "" && msg.Event != config.EventMsgDelete {
+	useWebhooks := b.shouldMessageUseWebhooks(&msg)
+	if useWebhooks && msg.Event != config.EventMsgDelete {
 		// skip events
 		if msg.Event != "" && msg.Event != config.EventUserAction && msg.Event != config.EventJoinLeave && msg.Event != config.EventTopicChange {
 			return "", nil
@@ -260,32 +264,18 @@ func (b *Bdiscord) Send(msg config.Message) (string, error) {
 
 		if msg.ID != "" {
 			b.Log.Debugf("Editing webhook message")
-			uri := discordgo.EndpointWebhookToken(wID, wToken) + "/messages/" + msg.ID
+			err := b.transmitter.Edit(channelID, msg.ID, &discordgo.WebhookParams{
-			_, err := b.c.RequestWithBucketID("PATCH", uri, discordgo.WebhookParams{
 				Content:  msg.Text,
 				Username: msg.Username,
-			}, discordgo.EndpointWebhookToken("", ""))
+			})
 			if err == nil {
 				return msg.ID, nil
 			}
 			b.Log.Errorf("Could not edit webhook message: %s", err)
 		}
 
-		b.Log.Debugf("Broadcasting using Webhook")
-
-		// if we have a global webhook for this Discord account, and permission
-		// to modify webhooks (previously verified), then set its channel to
-		// the message channel before using it.
-		if isGlobalWebhook && b.canEditWebhooks {
-			b.Log.Debugf("Setting webhook channel to \"%s\"", msg.Channel)
-			_, err := b.c.WebhookEdit(wID, "", "", channelID)
-			if err != nil {
-				b.Log.Errorf("Could not set webhook channel: %s", err)
-				return "", err
-			}
-		}
 		b.Log.Debugf("Processing webhook sending for message %#v", msg)
-		msg, err := b.webhookSend(&msg, wID, wToken)
+		msg, err := b.webhookSend(&msg, channelID)
 		if err != nil {
 			b.Log.Errorf("Could not broadcast via webook for message %#v: %s", msg, err)
 			return "", err
@@ -339,46 +329,6 @@ func (b *Bdiscord) Send(msg config.Message) (string, error) {
 	return res.ID, nil
 }
 
-// useWebhook returns true if we have a webhook defined somewhere
-func (b *Bdiscord) useWebhook() bool {
-	if b.GetString("WebhookURL") != "" {
-		return true
-	}
-
-	b.channelsMutex.RLock()
-	defer b.channelsMutex.RUnlock()
-
-	for _, channel := range b.channelInfoMap {
-		if channel.Options.WebhookURL != "" {
-			return true
-		}
-	}
-	return false
-}
-
-// isWebhookID returns true if the specified id is used in a defined webhook
-func (b *Bdiscord) isWebhookID(id string) bool {
-	if b.GetString("WebhookURL") != "" {
-		wID, _ := b.splitURL(b.GetString("WebhookURL"))
-		if wID == id {
-			return true
-		}
-	}
-
-	b.channelsMutex.RLock()
-	defer b.channelsMutex.RUnlock()
-
-	for _, channel := range b.channelInfoMap {
-		if channel.Options.WebhookURL != "" {
-			wID, _ := b.splitURL(channel.Options.WebhookURL)
-			if wID == id {
-				return true
-			}
-		}
-	}
-	return false
-}
-
 // handleUploadFile handles native upload of files
 func (b *Bdiscord) handleUploadFile(msg *config.Message, channelID string) (string, error) {
 	var err error
@@ -401,10 +351,26 @@ func (b *Bdiscord) handleUploadFile(msg *config.Message, channelID string) (stri
 	return "", nil
 }
 
+// shouldMessageUseWebhooks checks if have a channel specific webhook, if we're not using auto webhooks
+func (b *Bdiscord) shouldMessageUseWebhooks(msg *config.Message) bool {
+	if b.useAutoWebhooks {
+		return true
+	}
+
+	b.channelsMutex.RLock()
+	defer b.channelsMutex.RUnlock()
+	if ci, ok := b.channelInfoMap[msg.Channel+b.Account]; ok {
+		if ci.Options.WebhookURL != "" {
+			return true
+		}
+	}
+	return false
+}
+
 // webhookSend send one or more message via webhook, taking care of file
 // uploads (from slack, telegram or mattermost).
 // Returns messageID and error.
-func (b *Bdiscord) webhookSend(msg *config.Message, webhookID, token string) (*discordgo.Message, error) {
+func (b *Bdiscord) webhookSend(msg *config.Message, channelID string) (*discordgo.Message, error) {
 	var (
 		res *discordgo.Message
 		err error
@@ -427,10 +393,8 @@ func (b *Bdiscord) webhookSend(msg *config.Message, webhookID, token string) (*d
 
 	// We can't send empty messages.
 	if msg.Text != "" {
-		res, err = b.c.WebhookExecute(
+		res, err = b.transmitter.Send(
-			webhookID,
+			channelID,
-			token,
-			true,
 			&discordgo.WebhookParams{
 				Content:   msg.Text,
 				Username:  msg.Username,
@@ -454,10 +418,8 @@ func (b *Bdiscord) webhookSend(msg *config.Message, webhookID, token string) (*d
 			if msg.Text == "" {
 				content = fi.Comment
 			}
-			_, e2 := b.c.WebhookExecute(
+			_, e2 := b.transmitter.Send(
-				webhookID,
+				channelID,
-				token,
-				false,
 				&discordgo.WebhookParams{
 					Username:  msg.Username,
 					AvatarURL: msg.Avatar,

bridge/discord/handlers.go

@@ -69,7 +69,7 @@ func (b *Bdiscord) messageCreate(s *discordgo.Session, m *discordgo.MessageCreat
 		return
 	}
 	// if using webhooks, do not relay if it's ours
-	if b.useWebhook() && m.Author.Bot && b.isWebhookID(m.Author.ID) {
+	if m.Author.Bot && b.transmitter.HasWebhook(m.Author.ID) {
 		return
 	}
 

bridge/discord/helpers.go

@@ -196,7 +196,7 @@ func (b *Bdiscord) replaceAction(text string) (string, bool) {
 }
 
 // splitURL splits a webhookURL and returns the ID and token.
-func (b *Bdiscord) splitURL(url string) (string, string) {
+func (b *Bdiscord) splitURL(url string) (string, string, bool) {
 	const (
 		expectedWebhookSplitCount = 7
 		webhookIdxID              = 5
@@ -204,9 +204,9 @@ func (b *Bdiscord) splitURL(url string) (string, string) {
 	)
 	webhookURLSplit := strings.Split(url, "/")
 	if len(webhookURLSplit) != expectedWebhookSplitCount {
-		b.Log.Fatalf("%s is no correct discord WebhookURL", url)
+		return "", "", false
 	}
-	return webhookURLSplit[webhookIdxID], webhookURLSplit[webhookIdxToken]
+	return webhookURLSplit[webhookIdxID], webhookURLSplit[webhookIdxToken], true
 }
 
 func enumerateUsernames(s string) []string {

bridge/discord/transmitter/transmitter.go

@@ -0,0 +1,257 @@
+// Package transmitter provides functionality for transmitting
+// arbitrary webhook messages to Discord.
+//
+// The package provides the following functionality:
+// - Creating new webhooks, whenever necessary
+// - Loading webhooks that we have previously created
+// - Sending new messages
+// - Editing messages, via message ID
+// - Deleting messages, via message ID
+//
+// The package has been designed for matterbridge, but with other
+// Go bots in mind. The public API should be matterbridge-agnostic.
+package transmitter
+
+import (
+	"errors"
+	"fmt"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/matterbridge/discordgo"
+	log "github.com/sirupsen/logrus"
+)
+
+// A Transmitter represents a message manager for a single guild.
+type Transmitter struct {
+	session    *discordgo.Session
+	guild      string
+	title      string
+	autoCreate bool
+
+	// channelWebhooks maps from a channel ID to a webhook instance
+	channelWebhooks map[string]*discordgo.Webhook
+
+	mutex sync.RWMutex
+
+	Log *log.Entry
+}
+
+// ErrWebhookNotFound is returned when a valid webhook for this channel/message combination does not exist
+var ErrWebhookNotFound = errors.New("webhook for this channel and message does not exist")
+
+// ErrPermissionDenied is returned if the bot does not have permission to manage webhooks.
+//
+// It's important to note that:
+// - a bot can have both a guild-wide permission and a channel-specific permission to manage webhooks
+// - even if a bot has permission to manage the guild's webhooks, there could be channel specific overrides
+var ErrPermissionDenied = errors.New("missing 'Manage Webhooks' permission")
+
+// New returns a new Transmitter given a Discord session, guild ID, and title.
+func New(session *discordgo.Session, guild string, title string, autoCreate bool) *Transmitter {
+	return &Transmitter{
+		session:    session,
+		guild:      guild,
+		title:      title,
+		autoCreate: autoCreate,
+
+		channelWebhooks: make(map[string]*discordgo.Webhook),
+
+		Log: log.NewEntry(nil),
+	}
+}
+
+// Send transmits a message to the given channel with the provided webhook data.
+//
+// Note that this function will wait until Discord responds with an answer.
+func (t *Transmitter) Send(channelID string, params *discordgo.WebhookParams) (*discordgo.Message, error) {
+	wh, err := t.getOrCreateWebhook(channelID)
+	if err != nil {
+		return nil, err
+	}
+
+	msg, err := t.session.WebhookExecute(wh.ID, wh.Token, true, params)
+	if err != nil {
+		return nil, fmt.Errorf("execute failed: %w", err)
+	}
+
+	return msg, nil
+}
+
+// Edit will edit a message in a channel, if possible.
+func (t *Transmitter) Edit(channelID string, messageID string, params *discordgo.WebhookParams) error {
+	wh := t.getWebhook(channelID)
+
+	if wh == nil {
+		return ErrWebhookNotFound
+	}
+
+	uri := discordgo.EndpointWebhookToken(wh.ID, wh.Token) + "/messages/" + messageID
+	_, err := t.session.RequestWithBucketID("PATCH", uri, params, discordgo.EndpointWebhookToken("", ""))
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// HasWebhook checks whether the transmitter is using a particular webhook.
+func (t *Transmitter) HasWebhook(id string) bool {
+	t.mutex.RLock()
+	defer t.mutex.RUnlock()
+
+	for _, wh := range t.channelWebhooks {
+		if wh.ID == id {
+			return true
+		}
+	}
+
+	return false
+}
+
+// AddWebhook allows you to register a channel's webhook with the transmitter.
+func (t *Transmitter) AddWebhook(channelID string, webhook *discordgo.Webhook) (replaced bool) {
+	t.Log.Debugf("Manually added webhook %#v to channel %#v", webhook.ID, channelID)
+	t.mutex.Lock()
+	defer t.mutex.Unlock()
+
+	_, replaced := t.channelWebhooks[channelID]
+	t.channelWebhooks[channelID] = webhook
+	return replaced
+}
+
+// RefreshGuildWebhooks loads "relevant" webhooks into the transmitter, with careful permission handling.
+//
+// Notes:
+// - A webhook is "relevant" if it was created by this bot -- the ApplicationID should match the bot's ID.
+// - The term "having permission" means having the "Manage Webhooks" permission. See ErrPermissionDenied for more information.
+// - This function is additive and will not unload previously loaded webhooks.
+// - A nil channelIDs slice is treated the same as an empty one.
+//
+// If the bot has guild-wide permission:
+// 1. it will load any "relevant" webhooks from the entire guild
+// 2. the given slice is ignored
+//
+// If the bot does not have guild-wide permission:
+// 1. it will load any "relevant" webhooks in each channel
+// 2. a single error will be returned if any error occurs (incl. if there is no permission for any of these channels)
+//
+// If any channel has more than one "relevant" webhook, it will randomly pick one.
+func (t *Transmitter) RefreshGuildWebhooks(channelIDs []string) error {
+	t.Log.Debugln("Refreshing guild webhooks")
+
+	botID, err := getDiscordUserID(t.session)
+	if err != nil {
+		return fmt.Errorf("could not get current user: %w", err)
+	}
+
+	// Get all existing webhooks
+	hooks, err := t.session.GuildWebhooks(t.guild)
+	if err != nil {
+		switch {
+		case isDiscordPermissionError(err):
+			// We fallback on manually fetching hooks from individual channels
+			// if we don't have the "Manage Webhooks" permission globally.
+			// We can only do this if we were provided channelIDs, though.
+			if len(channelIDs) == 0 {
+				return ErrPermissionDenied
+			}
+			t.Log.Debugln("Missing global 'Manage Webhooks' permission, falling back on per-channel permission")
+			return t.fetchChannelsHooks(channelIDs, botID)
+		default:
+			return fmt.Errorf("could not get webhooks: %w", err)
+		}
+	}
+
+	t.Log.Debugln("Refreshing guild webhooks using global permission")
+	t.assignHooksByAppID(hooks, botID, false)
+	return nil
+}
+
+// createWebhook creates a webhook for a specific channel.
+func (t *Transmitter) createWebhook(channel string) (*discordgo.Webhook, error) {
+	t.mutex.Lock()
+	defer t.mutex.Unlock()
+
+	wh, err := t.session.WebhookCreate(channel, t.title+time.Now().Format(" 3:04:05PM"), "")
+	if err != nil {
+		return nil, err
+	}
+
+	t.channelWebhooks[channel] = wh
+	return wh, nil
+}
+
+func (t *Transmitter) getWebhook(channel string) *discordgo.Webhook {
+	t.mutex.RLock()
+	defer t.mutex.RUnlock()
+
+	return t.channelWebhooks[channel]
+}
+
+func (t *Transmitter) getOrCreateWebhook(channelID string) (*discordgo.Webhook, error) {
+	// If we have a webhook for this channel, immediately return it
+	wh := t.getWebhook(channelID)
+	if wh != nil {
+		return wh, nil
+	}
+
+	// Early exit if we don't want to automatically create one
+	if !t.autoCreate {
+		return nil, ErrWebhookNotFound
+	}
+
+	t.Log.Infof("Creating a webhook for %s\n", channelID)
+	wh, err := t.createWebhook(channelID)
+	if err != nil {
+		return nil, fmt.Errorf("could not create webhook: %w", err)
+	}
+
+	return wh, nil
+}
+
+// fetchChannelsHooks fetches hooks for the given channelIDs and calls assignHooksByAppID for each channel's hooks
+func (t *Transmitter) fetchChannelsHooks(channelIDs []string, botID string) error {
+	// For each channel, search for relevant hooks
+	var failedHooks []string
+	for _, channelID := range channelIDs {
+		hooks, err := t.session.ChannelWebhooks(channelID)
+		if err != nil {
+			failedHooks = append(failedHooks, "\n- "+channelID+": "+err.Error())
+			continue
+		}
+		t.assignHooksByAppID(hooks, botID, true)
+	}
+
+	// Compose an error if any hooks failed
+	if len(failedHooks) > 0 {
+		return errors.New("failed to fetch hooks:" + strings.Join(failedHooks, ""))
+	}
+
+	return nil
+}
+
+func (t *Transmitter) assignHooksByAppID(hooks []*discordgo.Webhook, appID string, channelTargeted bool) {
+	logLine := "Picking up webhook"
+	if channelTargeted {
+		logLine += " (channel targeted)"
+	}
+
+	t.mutex.Lock()
+	defer t.mutex.Unlock()
+
+	for _, wh := range hooks {
+		if wh.ApplicationID != appID {
+			continue
+		}
+
+		t.channelWebhooks[wh.ChannelID] = wh
+		t.Log.WithFields(log.Fields{
+			"id":      wh.ID,
+			"name":    wh.Name,
+			"channel": wh.ChannelID,
+		}).Println(logLine)
+		break
+	}
+}

bridge/discord/transmitter/utils.go

@@ -0,0 +1,32 @@
+package transmitter
+
+import (
+	"github.com/matterbridge/discordgo"
+)
+
+// isDiscordPermissionError returns false for nil, and true if a Discord RESTError with code discordgo.ErrorCodeMissionPermissions
+func isDiscordPermissionError(err error) bool {
+	if err == nil {
+		return false
+	}
+
+	restErr, ok := err.(*discordgo.RESTError)
+	if !ok {
+		return false
+	}
+
+	return restErr.Message != nil && restErr.Message.Code == discordgo.ErrCodeMissingPermissions
+}
+
+// getDiscordUserID gets own user ID from state, and fallback on API request
+func getDiscordUserID(session *discordgo.Session) (string, error) {
+	if user := session.State.User; user != nil {
+		return user.ID, nil
+	}
+
+	user, err := session.User("@me")
+	if err != nil {
+		return "", err
+	}
+	return user.ID, nil
+}