jshiffer/matterbridge
1
0
Fork 1
mirror of https://github.com/42wim/matterbridge.git synced 2024-11-24 03:32:02 -08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Optimize handling of very large slack teams. Fixes #695

Browse Source 
Stop getting users if we reach 2000 users. Slack will rate-limit us
even if we follow their limits.
This means that we now have to lookup every user that says a message
for the first time. This should be less intensive on the API.

This also disables partly fb713ed91b for now.
ChannelMembers will not be filled.
This commit is contained in:
Wim 2019-01-30 23:26:36 +01:00
parent 707db950c8
commit 4ae028fe73
2 changed files with 27 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
bridge/slack/handlers.go
View File

@ -75,8 +75,6 @@ func (b *Bslack) handleSlackClient(messages chan *config.Message) {
// When we join a channel we update the full list of users as
// well as the information for the channel that we joined as this
// should now tell that we are a member of it.
b.populateUsers(false)
b.channelsMutex.Lock()
b.channelsByID[ev.Channel.ID] = &ev.Channel
b.channelsByName[ev.Channel.Name] = &ev.Channel
@ -202,7 +200,6 @@ func (b *Bslack) handleMessageEvent(ev *slack.MessageEvent) (*config.Message, er
func (b *Bslack) handleStatusEvent(ev *slack.MessageEvent, rmsg *config.Message) bool {
switch ev.SubType {
case sChannelJoined, sMemberJoined:
b.populateUsers(false)
// There's no further processing needed on channel events
// so we return 'true'.
return true

19
bridge/slack/helpers.go
View File

@ -12,6 +12,13 @@ import (
)
func (b *Bslack) getUser(id string) *slack.User {
b.usersMutex.RLock()
user, ok := b.users[id]
b.usersMutex.RUnlock()
if ok {
return user
}
b.populateUser(id)
b.usersMutex.RLock()
defer b.usersMutex.RUnlock()
@ -102,9 +109,11 @@ func (b *Bslack) populateUsers(wait bool) {
newUsers := map[string]*slack.User{}
pagination := b.sc.GetUsersPaginated(slack.GetUsersOptionLimit(200))
count := 0
for {
var err error
pagination, err = pagination.Next(context.Background())
time.Sleep(time.Second)
if err != nil {
if pagination.Done(err) {
break
@ -120,6 +129,13 @@ func (b *Bslack) populateUsers(wait bool) {
for i := range pagination.Users {
newUsers[pagination.Users[i].ID] = &pagination.Users[i]
}
b.Log.Debugf("getting %d users", len(pagination.Users))
count++
// more > 2000 users, slack will complain and ratelimit. break
if count > 10 {
b.Log.Info("Large slack detected > 2000 users, skipping loading complete userlist.")
break
}
}
b.usersMutex.Lock()
@ -172,6 +188,8 @@ func (b *Bslack) populateChannels(wait bool) {
newChannelsByID[channels[i].ID] = &channels[i]
newChannelsByName[channels[i].Name] = &channels[i]
// also find all the members in every channel
// comment for now, issues on big slacks
/*
members, err := b.getUsersInConversation(channels[i].ID)
if err != nil {
if err = b.handleRateLimit(err); err != nil {
@ -181,6 +199,7 @@ func (b *Bslack) populateChannels(wait bool) {
continue
}
newChannelMembers[channels[i].ID] = members
*/
}
if nextCursor == "" {